rouge 3.23.0 → 3.25.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: abcc9751ef5e076b3765942693a8f098d83f746aa7527b683bf1b5772f700b48
-  data.tar.gz: 06074e941ba684cbe5871dfcaeb1dafca415cd228e7223c679847a137d937693
+  metadata.gz: 35d2eb9627935d1f4e510d66263bddb55773667d8a441677019a377abaf3784e
+  data.tar.gz: 04370e4428e5f4c30a552d455bbf5655beeae8bf3ddc68db09ba0fbf76fee3e9
 SHA512:
-  metadata.gz: 2db089d9c952c5d79a93659bcb95f008630a60ea35b756f905a51f6d583dc7ab17470faa1a17f77c1165fa0ecee495c0d890a2bebdcc4aef5f03465620691283
-  data.tar.gz: 8a161de0a32d5a3be6ae31d4112949bea3ffdbd9bd6408dc6b4a6fa3359d9b34c4b214c383c71f3927c9ed2f2431e1240de9cf0f4f02f9d098ebad1ba367dd66
+  metadata.gz: fc847dd05c6d47cdbcc4296b60594faf8477b1c143e8f303c84a09bfbd88345baa5dbe19e4238ef6a37c79e59809616f649679070faf17f556838ce874cb95b9
+  data.tar.gz: 272f4948d1935e9e9c1eaae8e5230e5db42933dcf1e2130dad009d6d3e4c57fee656b4aead64d6933cadf836bd7d82287fbc8193032c1af7ebf21e53fc7fb403

data/lib/rouge.rb

@@ -83,6 +83,7 @@ Rouge.load_file 'formatters/html_table'
 Rouge.load_file 'formatters/html_pygments'
 Rouge.load_file 'formatters/html_legacy'
 Rouge.load_file 'formatters/html_linewise'
+Rouge.load_file 'formatters/html_line_highlighter'
 Rouge.load_file 'formatters/html_line_table'
 Rouge.load_file 'formatters/html_inline'
 Rouge.load_file 'formatters/terminal256'

data/lib/rouge/cli.rb

@@ -44,6 +44,7 @@ module Rouge
       yield %||
       yield %|where <command> is one of:|
       yield %|	highlight	#{Highlight.desc}|
+      yield %|	debug		#{Debug.desc}|
       yield %|	help		#{Help.desc}|
       yield %|	style		#{Style.desc}|
       yield %|	list		#{List.desc}|
@@ -104,6 +105,8 @@ module Rouge
         Help
       when 'highlight', 'hi'
         Highlight
+      when 'debug'
+        Debug
       when 'style'
         Style
       when 'list'
@@ -215,7 +218,7 @@ module Rouge
         end
       end
 
-      def self.parse(argv)
+      def self.parse_opts(argv)
         opts = {
           :formatter => supports_truecolor? ? 'terminal-truecolor' : 'terminal256',
           :theme => 'thankful_eyes',
@@ -256,7 +259,11 @@ module Rouge
           end
         end
 
-        new(opts)
+        opts
+      end
+
+      def self.parse(argv)
+        new(parse_opts(argv))
       end
 
       def input_stream
@@ -344,6 +351,29 @@ module Rouge
       end
     end
 
+    class Debug < Highlight
+      def self.desc
+      end
+
+      def self.doc
+        return enum_for(:doc) unless block_given?
+
+        yield %|usage: rougify debug [<options>]|
+        yield %||
+        yield %|Debug a lexer. Similar options to `rougify highlight`, but|
+        yield %|defaults to the `null` formatter, and ensures the `debug`|
+        yield %|option is enabled, to print debugging information to stdout.|
+      end
+
+      def self.parse_opts(argv)
+        out = super(argv)
+        out[:lexer_opts]['debug'] = '1'
+        out[:formatter] = 'null'
+
+        out
+      end
+    end
+
     class Style < CLI
       def self.desc
         "print CSS styles"

data/lib/rouge/demos/email

@@ -0,0 +1,11 @@
+From: Me <me@example.com>
+To: You <you@example.com>
+Date: Tue, 21 Jul 2020 15:14:03 +0000
+Subject: A very important message
+
+> Please investigate.  Thank you.
+
+I have investigated.
+
+-- 
+This message is highly confidential and will self-destruct.

data/lib/rouge/demos/j

@@ -0,0 +1,12 @@
+NB. Procedural programming
+fizzbuzz=: monad define
+  for_i. >:i.y do.
+    if.     0 = 15 | i do. echo'FizzBuzz'
+    elseif. 0 =  3 | i do. echo'Fizz'
+    elseif. 0 =  5 | i do. echo'Buzz'
+    else.                  echo i
+    end.
+  end.
+)
+NB. Loopless programming
+fizzbuzz=: echo@(, ::] ('Fizz' ; 'Buzz') ;@#~ 0 = 3 5&|)@>:@i.

data/lib/rouge/formatters/html_line_highlighter.rb

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Formatters
+    class HTMLLineHighlighter < Formatter
+      tag 'html_line_highlighter'
+
+      def initialize(delegate, opts = {})
+        @delegate = delegate
+        @highlight_line_class = opts.fetch(:highlight_line_class, 'hll')
+        @highlight_lines = opts[:highlight_lines] || []
+      end
+
+      def stream(tokens)
+        token_lines(tokens).with_index(1) do |line_tokens, lineno|
+          line = %(#{@delegate.format(line_tokens)}\n)
+          line = %(<span class="#{@highlight_line_class}">#{line}</span>) if @highlight_lines.include? lineno
+          yield line
+        end
+      end
+    end
+  end
+end

data/lib/rouge/formatters/html_line_table.rb

@@ -32,10 +32,8 @@ module Rouge
       end
 
       def stream(tokens, &b)
-        lineno = @start_line - 1
         buffer = [%(<table class="#@table_class"><tbody>)]
-        token_lines(tokens) do |line_tokens|
-          lineno += 1
+        token_lines(tokens).with_index(@start_line) do |line_tokens, lineno|
           buffer << %(<tr id="#{sprintf @line_id, lineno}" class="#@line_class">)
           buffer << %(<td class="#@gutter_class gl" )
           buffer << %(style="-moz-user-select: none;-ms-user-select: none;)

data/lib/rouge/formatters/html_linewise.rb

@@ -11,9 +11,8 @@ module Rouge
       end
 
       def stream(tokens, &b)
-        lineno = 0
-        token_lines(tokens) do |line_tokens|
-          yield %(<#{@tag_name} class="#{sprintf @class_format, lineno += 1}">)
+        token_lines(tokens).with_index(1) do |line_tokens, lineno|
+          yield %(<#{@tag_name} class="#{sprintf @class_format, lineno}">)
           @formatter.stream(line_tokens) {|formatted| yield formatted }
           yield %(\n</#{@tag_name}>)
         end

data/lib/rouge/lexer.rb

@@ -38,28 +38,15 @@ module Rouge
         registry[name.to_s]
       end
 
-      # Find a lexer, with fancy shiny features.
-      #
-      # * The string you pass can include CGI-style options
-      #
-      #     Lexer.find_fancy('erb?parent=tex')
-      #
-      # * You can pass the special name 'guess' so we guess for you,
-      #   and you can pass a second argument of the code to guess by
-      #
-      #     Lexer.find_fancy('guess', "#!/bin/bash\necho Hello, world")
+      # Same as ::find_fancy, except instead of returning an instantiated
+      # lexer, returns a pair of [lexer_class, options], so that you can
+      # modify or provide additional options to the lexer.
       #
-      #   If the code matches more than one lexer then Guesser::Ambiguous
-      #   is raised.
-      #
-      # This is used in the Redcarpet plugin as well as Rouge's own
-      # markdown lexer for highlighting internal code blocks.
-      #
-      def find_fancy(str, code=nil, additional_options={})
-
+      # Please note: the lexer class might be nil!
+      def lookup_fancy(str, code=nil, default_options={})
         if str && !str.include?('?') && str != 'guess'
           lexer_class = find(str)
-          return lexer_class && lexer_class.new(additional_options)
+          return [lexer_class, default_options]
         end
 
         name, opts = str ? str.split('?', 2) : [nil, '']
@@ -75,7 +62,7 @@ module Rouge
           [ k.to_s, val ]
         end
 
-        opts = additional_options.merge(Hash[opts])
+        opts = default_options.merge(Hash[opts])
 
         lexer_class = case name
         when 'guess', nil
@@ -84,6 +71,29 @@ module Rouge
           self.find(name)
         end
 
+        [lexer_class, opts]
+      end
+
+      # Find a lexer, with fancy shiny features.
+      #
+      # * The string you pass can include CGI-style options
+      #
+      #     Lexer.find_fancy('erb?parent=tex')
+      #
+      # * You can pass the special name 'guess' so we guess for you,
+      #   and you can pass a second argument of the code to guess by
+      #
+      #     Lexer.find_fancy('guess', "#!/bin/bash\necho Hello, world")
+      #
+      #   If the code matches more than one lexer then Guesser::Ambiguous
+      #   is raised.
+      #
+      # This is used in the Redcarpet plugin as well as Rouge's own
+      # markdown lexer for highlighting internal code blocks.
+      #
+      def find_fancy(str, code=nil, default_options={})
+        lexer_class, opts = lookup_fancy(str, code, default_options)
+
         lexer_class && lexer_class.new(opts)
       end
 
@@ -317,6 +327,14 @@ module Rouge
       @debug = Lexer.debug_enabled? && bool_option('debug')
     end
 
+    # Returns a new lexer with the given options set. Useful for e.g. setting
+    # debug flags post hoc, or providing global overrides for certain options
+    def with(opts={})
+      new_options = @options.dup
+      opts.each { |k, v| new_options[k.to_s] = v }
+      self.class.new(new_options)
+    end
+
     def as_bool(val)
       case val
       when nil, false, 0, '0', 'false', 'off'

data/lib/rouge/lexers/apex.rb

@@ -49,7 +49,7 @@ module Rouge
 
       state :root do
         rule %r/\s+/m, Text
-        
+
         rule %r(//.*?$), Comment::Single
         rule %r(/\*.*?\*/)m, Comment::Multiline
 
@@ -57,17 +57,19 @@ module Rouge
         rule %r/import\b/, Keyword::Namespace, :import
 
         rule %r/([@$.]?)(#{id})([:(]?)/io do |m|
-          if self.class.keywords.include? m[0].downcase
+          lowercased = m[0].downcase
+          uppercased = m[0].upcase
+          if self.class.keywords.include? lowercased
             token Keyword
-          elsif self.class.soql.include? m[0].upcase
+          elsif self.class.soql.include? uppercased
             token Keyword
-          elsif self.class.declarations.include? m[0].downcase
+          elsif self.class.declarations.include? lowercased
             token Keyword::Declaration
-          elsif self.class.types.include? m[0].downcase
+          elsif self.class.types.include? lowercased
             token Keyword::Type
-          elsif self.class.constants.include? m[0].downcase
+          elsif self.class.constants.include? lowercased
             token Keyword::Constant
-          elsif 'package'.casecmp m[0]
+          elsif lowercased == 'package'
             token Keyword::Namespace
           elsif m[1] == "@"
             token Name::Decorator

data/lib/rouge/lexers/batchfile.rb

@@ -79,6 +79,7 @@ module Rouge
       state :basic do
         # Comments
         rule %r/@?\brem\b.*$/i, Comment
+
         # Empty Labels
         rule %r/^::.*$/, Comment
 
@@ -105,7 +106,7 @@ module Rouge
           end
         end
 
-        rule %r/([\/\-+][a-z]+)\s*/i, Name::Attribute
+        rule %r/((?:[\/\+]|--?)[a-z]+)\s*/i, Name::Attribute
 
         mixin :expansions
 

data/lib/rouge/lexers/cpp.rb

@@ -63,7 +63,7 @@ module Rouge
         rule %r((#{dq}[.]#{dq}?|[.]#{dq})(e[+-]?#{dq}[lu]*)?)i, Num::Float
         rule %r(#{dq}e[+-]?#{dq}[lu]*)i, Num::Float
         rule %r/0x\h('?\h)*[lu]*/i, Num::Hex
-        rule %r/0b[01]+(?:_[01]+)*/, Num::Bin
+        rule %r/0b[01]+('[01]+)*/, Num::Bin
         rule %r/0[0-7]('?[0-7])*[lu]*/i, Num::Oct
         rule %r/#{dq}[lu]*/i, Num::Integer
         rule %r/\bnullptr\b/, Name::Builtin

data/lib/rouge/lexers/docker.rb

@@ -7,7 +7,7 @@ module Rouge
       title "Docker"
       desc "Dockerfile syntax"
       tag 'docker'
-      aliases 'dockerfile'
+      aliases 'dockerfile', 'Dockerfile'
       filenames 'Dockerfile', '*.Dockerfile', '*.docker'
       mimetypes 'text/x-dockerfile-config'
 

data/lib/rouge/lexers/elm.rb

@@ -78,13 +78,13 @@ module Rouge
         rule %r/"/, Str::Double, :pop!
       end
 
-      # Multiple line string with tripple double quotes, e.g. """ multi """
+      # Multiple line string with triple double quotes, e.g. """ multi """
       state :multiline_string do
-        rule %r/\s*"""/, Str, :pop!
-        rule %r/.*/, Str
-        rule %r/\s*/, Str
+        rule %r/\\"/, Str::Escape
+        rule %r/"""/, Str, :pop!
+        rule %r/[^"]+/, Str
+        rule %r/"/, Str
       end
-
     end
   end
 end

data/lib/rouge/lexers/email.rb

@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    class Email < RegexLexer
+      tag 'email'
+      aliases 'eml', 'e-mail'
+      filenames '*.eml'
+      mimetypes 'message/rfc822'
+
+      title "Email"
+      desc "An email message"
+
+      start do
+        push :fields
+      end
+
+      state :fields do
+        rule %r/[:]/, Operator, :field_body
+        rule %r/[^\n\r:]+/, Name::Tag
+        rule %r/[\n\r]/, Name::Tag
+      end
+
+      state :field_body do
+        rule(/(\r?\n){2}/) { token Text; pop!(2) }
+        rule %r/\r?\n(?![ \v\t\f])/, Text, :pop!
+        rule %r/[^\n\r]+/, Name::Attribute
+        rule %r/[\n\r]/, Name::Attribute
+      end
+
+      state :root do
+        rule %r/\n/, Text
+        rule %r/^>.*/, Comment
+        rule %r/.+/, Text
+      end
+    end
+  end
+end

data/lib/rouge/lexers/ghc_core.rb

@@ -27,8 +27,9 @@ module Rouge
         mixin :function
 
         # rest is Text
+        # TODO: this is really inefficient
         rule %r/\s/m, Text
-        rule %r/.*/, Text
+        rule %r/./, Text
       end
 
       state :expression do

data/lib/rouge/lexers/graphql.rb

@@ -20,7 +20,7 @@ module Rouge
         end
 
         rule %r/\bfragment\b/, Keyword, :fragment_definition
-        
+
         rule %r/\bscalar\b/, Keyword, :value
 
         rule %r/\b(?:type|interface|enum)\b/, Keyword, :type_definition

data/lib/rouge/lexers/http.rb

@@ -8,16 +8,22 @@ module Rouge
       title "HTTP"
       desc 'http requests and responses'
 
+      option :content, "the language for the content (default: auto-detect)"
+
       def self.http_methods
         @http_methods ||= %w(GET POST PUT DELETE HEAD OPTIONS TRACE PATCH)
       end
 
       def content_lexer
+        @content_lexer ||= (lexer_option(:content) || guess_content_lexer)
+      end
+
+      def guess_content_lexer
         return Lexers::PlainText unless @content_type
 
-        @content_lexer ||= Lexer.guess_by_mimetype(@content_type)
+        Lexer.guess_by_mimetype(@content_type)
       rescue Lexer::AmbiguousGuess
-        @content_lexer = Lexers::PlainText
+        Lexers::PlainText
       end
 
       start { @content_type = 'text/plain' }

data/lib/rouge/lexers/isbl.rb

@@ -41,7 +41,7 @@ module Rouge
 
       state :dotted do
         mixin :whitespace
-        rule %r/[a-zа-яё_0-9]*/i do |m|
+        rule %r/[a-zа-яё_0-9]+/i do |m|
           name = m[0]
           if self.class.constants.include? name.downcase
             token Name::Builtin
@@ -56,7 +56,7 @@ module Rouge
 
       state :type do
         mixin :whitespace
-        rule %r/[a-zа-яё_0-9]*/i do |m|
+        rule %r/[a-zа-яё_0-9]+/i do |m|
           name = m[0]
           if self.class.interfaces.include? name.downcase
             token Keyword::Type

data/lib/rouge/lexers/j.rb

@@ -0,0 +1,244 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    class J < RegexLexer
+      title 'J'
+      desc "The J programming language (jsoftware.com)"
+      tag 'j'
+      filenames '*.ijs', '*.ijt'
+
+      # For J-specific terms we use, see:
+      #   https://code.jsoftware.com/wiki/Vocabulary/AET
+      #   https://code.jsoftware.com/wiki/Vocabulary/Glossary
+
+      # https://code.jsoftware.com/wiki/Vocabulary/PartsOfSpeech
+      def self.token_map
+        @token_map ||= {
+          noun: Keyword::Constant,
+          verb: Name::Function,
+          modifier: Operator,
+          name: Name,
+          param: Name::Builtin::Pseudo,
+          other: Punctuation,
+          nil => Error,
+        }
+      end
+
+      # https://code.jsoftware.com/wiki/NuVoc
+      def self.inflection_list
+        @inflection_list ||= ['', '.', ':', '..', '.:', ':.', '::']
+      end
+
+      def self.primitive_table
+        @primitive_table ||= Hash.new([:name]).tap do |h|
+          {
+            '()' => [:other],
+            '=' => [:verb, :other, :other],
+            '<>+-*%$|,#' => [:verb, :verb, :verb],
+            '^' => [:verb, :verb, :modifier],
+            '~"' => [:modifier, :verb, :verb],
+            '.:@' => [:modifier, :modifier, :modifier],
+            ';' => [:verb, :modifier, :verb],
+            '!' => [:verb, :modifier, :modifier],
+            '/\\' => [:modifier, :modifier, :verb],
+            '[' => [:verb, nil, :verb],
+            ']' => [:verb],
+            '{' => [:verb, :verb, :verb, nil, nil, nil, :verb],
+            '}' => [:modifier, :verb, :verb, nil, nil, nil, :modifier],
+            '`' => [:modifier, nil, :modifier],
+            '&' => [:modifier, :modifier, :modifier, nil, :modifier],
+            '?' => [:verb, :verb],
+            'a' => [:name, :noun, :noun],
+            'ACeEIjorv' => [:name, :verb],
+            'bdfHMT' => [:name, :modifier],
+            'Dt' => [:name, :modifier, :modifier],
+            'F' => [:name, :modifier, :modifier, :modifier, :modifier,
+                    :modifier, :modifier],
+            'iu' => [:name, :verb, :verb],
+            'L' => [:name, :verb, :modifier],
+            'mny' => [:param],
+            'p' => [:name, :verb, :verb, :verb],
+            'qsZ' => [:name, nil, :verb],
+            'S' => [:name, nil, :modifier],
+            'u' => [:param, :verb, :verb],
+            'v' => [:param, :verb],
+            'x' => [:param, nil, :verb],
+          }.each {|k, v| k.each_char {|c| h[c] = v } }
+        end
+      end
+
+      def self.primitive(char, inflection)
+        i = inflection_list.index(inflection) or return Error
+        token_map[primitive_table[char][i]]
+      end
+
+      def self.control_words
+        @control_words ||= Set.new %w(
+          assert break case catch catchd catcht continue do else elseif end
+          fcase for if return select throw try while whilst
+        )
+      end
+
+      def self.control_words_id
+        @control_words_id ||= Set.new %w(for goto label)
+      end
+
+      state :expr do
+        rule %r/\s+/, Text
+
+        rule %r'([!-&(-/:-@\[-^`{-~]|[A-Za-z]\b)([.:]*)' do |m|
+          token J.primitive(m[1], m[2])
+        end
+
+        rule %r/(?:\d|_\d?):([.:]*)/ do |m|
+          token m[1].empty? ? J.token_map[:verb] : Error
+        end
+
+        rule %r/[\d_][\w.]*([.:]*)/ do |m|
+          token m[1].empty? ? Num : Error
+        end
+
+        rule %r/'/, Str::Single, :str
+
+        rule %r/NB\.(?![.:]).*/, Comment::Single
+
+        rule %r/([A-Za-z]\w*)([.:]*)/ do |m|
+          if m[2] == '.'
+            word, sep, id = m[1].partition '_'
+            list = if sep.empty?
+              J.control_words
+            elsif not id.empty?
+              J.control_words_id
+            end
+            if list and list.include? word
+              token Keyword, word + sep
+              token((word == 'for' ? Name : Name::Label), id)
+              token Keyword, m[2]
+            else
+              token Error
+            end
+          else
+            token m[2].empty? ? Name : Error
+          end
+        end
+      end
+
+      state :str do
+        rule %r/''/, Str::Escape
+        rule %r/[^'\n]+/, Str::Single
+        rule %r/'|$/, Str::Single, :pop!
+      end
+
+      start do
+        @note_next = false
+      end
+
+      state :root do
+        rule %r/\n/ do
+          token Text
+          if @note_next
+            push :note
+            @note_next = false
+          end
+        end
+
+        # https://code.jsoftware.com/wiki/Vocabulary/com
+        # https://code.jsoftware.com/wiki/Vocabulary/NounExplicitDefinition
+        rule %r/
+          ([0-4]|13|adverb|conjunction|dyad|monad|noun|verb)([\ \t]+)
+          (def(?:ine)?\b|:)(?![.:])([\ \t]*)
+        /x do |m|
+          groups Keyword::Pseudo, Text, Keyword::Pseudo, Text
+          @def_body = (m[1] == '0' || m[1] == 'noun') ? :noun : :code
+          if m[3] == 'define'
+            # stack: [:root]
+            #    or  [:root, ..., :def_next]
+            pop! if stack.size > 1
+            push @def_body
+            push :def_next  # [:root, ..., @def_body, :def_next]
+          else
+            push :expl_def
+          end
+        end
+
+        rule %r/^([ \t]*)(Note\b(?![.:]))([ \t\r]*)(?!=[.:]|$)/ do
+          groups Text, Name, Text
+          @note_next = true
+        end
+
+        rule %r/[mnuvxy]\b(?![.:])/, Name
+        mixin :expr
+      end
+
+      state :def_next do
+        rule %r/\n/, Text, :pop!
+        mixin :root
+      end
+
+      state :expl_def do
+        rule %r/0\b(?![.:])/ do
+          token Keyword::Pseudo
+          # stack: [:root, :expl_def]
+          #    or  [:root, ..., :def_next, :expl_def]
+          pop! if stack.size > 2
+          goto @def_body
+          push :def_next  # [:root, ..., @def_body, :def_next]
+        end
+        rule %r/'/ do
+          if @def_body == :noun
+            token Str::Single
+            goto :str
+          else
+            token Punctuation
+            goto :q_expr
+          end
+        end
+        rule(//) { pop! }
+      end
+
+      # `q_expr` lexes the content of a string literal which is a part of an
+      # explicit definition.
+      # e.g. dyad def 'x + y'
+      state :q_expr do
+        rule %r/''/, Str::Single, :q_str
+        rule %r/'|$/, Punctuation, :pop!
+        rule %r/NB\.(?![.:])([^'\n]|'')*/, Comment::Single
+        mixin :expr
+      end
+
+      state :q_str do
+        rule %r/''''/, Str::Escape
+        rule %r/[^'\n]+/, Str::Single
+        rule %r/''/, Str::Single, :pop!
+        rule(/'|$/) { token Punctuation; pop! 2 }
+      end
+
+      state :note do
+        mixin :delimiter
+        rule %r/.+\n?/, Comment::Multiline
+      end
+
+      state :noun do
+        mixin :delimiter
+        rule %r/.+\n?/, Str::Heredoc
+      end
+
+      state :code do
+        mixin :delimiter
+        rule %r/^([ \t]*)(:)([ \t\r]*)$/ do
+          groups Text, Punctuation, Text
+        end
+        mixin :expr
+      end
+
+      state :delimiter do
+        rule %r/^([ \t]*)(\))([ \t\r]*$\n?)/ do
+          groups Text, Punctuation, Text
+          pop!
+        end
+      end
+    end
+  end
+end

data/lib/rouge/lexers/janet.rb

@@ -114,7 +114,7 @@ module Rouge
         end
       end
 
-      punctuation = %r/[_!@$%^&*+=~<>.?\/-]/o
+      punctuation = %r/[_!$%^&*+=~<>.?\/-]/o
       symbol = %r/([[:alpha:]]|#{punctuation})([[:word:]]|#{punctuation}|:)*/o
 
       state :root do
@@ -143,20 +143,19 @@ module Rouge
 
         rule %r/\(/, Punctuation, :function
 
-        rule %r/(')([\(\[])/ do
+        rule %r/(')(@?[(\[{])/ do
           groups Operator, Punctuation
           push :quote
         end
 
-        rule %r/(~)([\(\[])/ do
+        rule %r/(~)(@?[(\[{])/ do
           groups Operator, Punctuation
           push :quasiquote
         end
 
         rule %r/[\#~,';\|]/, Operator
 
-        rule %r/@?[({\[]/, Punctuation, :push
-        rule %r/[)}\]]/, Punctuation, :pop!
+        rule %r/@?[(){}\[\]]/, Punctuation
 
         rule symbol, Name
       end
@@ -169,6 +168,8 @@ module Rouge
       end
 
       state :function do
+        rule %r/[\)]/, Punctuation, :pop!
+
         rule symbol do |m|
           case m[0]
           when "quote"
@@ -187,8 +188,9 @@ module Rouge
       end
 
       state :quote do
-        rule %r/[\(\[]/, Punctuation, :push
-        rule symbol, Str::Symbol
+        rule %r/[(\[{]/, Punctuation, :push
+        rule %r/[)\]}]/, Punctuation, :pop!
+        rule symbol, Str::Escape
         mixin :root
       end
 
@@ -199,7 +201,6 @@ module Rouge
         end
         rule %r/(\()(\s*)(unquote)(\s+)(\()/ do
           groups Punctuation, Text, Keyword, Text, Punctuation
-          push :root
           push :function
         end
 

data/lib/rouge/lexers/javascript.rb

@@ -162,6 +162,10 @@ module Rouge
           push :template_string
         end
 
+        # special case for the safe navigation operator ?.
+        # so that we don't start detecting a ternary expr
+        rule %r/[?][.]/, Punctuation
+
         rule %r/[?]/ do
           token Punctuation
           push :ternary
@@ -265,9 +269,11 @@ module Rouge
 
       # template strings
       state :template_string do
-        rule %r/\${/, Punctuation, :template_string_expr
+        rule %r/[$]{/, Punctuation, :template_string_expr
         rule %r/`/, Str::Double, :pop!
-        rule %r/(\\\\|\\[\$`]|[^\$`]|\$(?!{))*/, Str::Double
+        rule %r/\\[$`]/, Str::Escape
+        rule %r/[^$`\\]+/, Str::Double
+        rule %r/[\\$]/, Str::Double
       end
 
       state :template_string_expr do

data/lib/rouge/lexers/jsl.rb

@@ -48,7 +48,7 @@ module Rouge
         rule %r/\\![btrnNf0\\"]/, Str::Escape
         rule %r/\\/, Str::Double
         rule %r/"/, Str::Double, :pop!
-        rule %r/[^\\"]*/m, Str::Double
+        rule %r/[^\\"]+/m, Str::Double
       end
     end
   end

data/lib/rouge/lexers/jsonnet.rb

@@ -126,6 +126,7 @@ module Rouge
 
       state :string do
         rule %r/\\([\\\/bfnrt]|(u[0-9a-fA-F]{4}))/, Str::Escape
+        rule %r/\\./, Str::Escape
       end
 
       state :string_double do
@@ -137,15 +138,15 @@ module Rouge
 
       state :string_single do
         mixin :string
-        rule %r/\\'/, Str::Escape
         rule %r/'/, Str, :pop!
         rule %r/[^\\']+/, Str
       end
 
       state :string_block do
         mixin :string
-        rule %r/\|\|\|/, Str, :pop!
-        rule %r/.*/, Str
+        rule %r/[|][|][|]/, Str, :pop!
+        rule %r/[^|\\]+/, Str
+        rule %r/[|]/, Str
       end
     end
   end

data/lib/rouge/lexers/jsp.rb

@@ -112,9 +112,8 @@ module Rouge
 
       state :jsp_interp_literal_start do
         rule %r/'/, Literal, :pop!
-        rule %r/[^']*/, Literal
+        rule %r/[^']+/, Literal
       end
-
     end
   end
-end
+end

data/lib/rouge/lexers/kotlin.rb

@@ -53,7 +53,7 @@ module Rouge
           groups Keyword::Declaration, Text
           push :property
         end
-        rule %r'(return|continue|break|this|super)(@#{name})?' do
+        rule %r'(return|continue|break|this|super)(@#{name})?\b' do
           groups Keyword, Name::Decorator
         end
         rule %r'\bfun\b', Keyword
@@ -133,9 +133,10 @@ module Rouge
       end
 
       state :comment do
-        rule %r'\s*/[*].*', Comment::Multiline, :comment
-        rule %r'.*[*]/', Comment::Multiline, :pop!
-        rule %r'.*', Comment::Multiline
+        rule %r'/[*]', Comment::Multiline, :comment
+        rule %r'[*]/', Comment::Multiline, :pop!
+        rule %r'[^/*]+', Comment::Multiline
+        rule %r'[/*]', Comment::Multiline
       end
     end
   end

data/lib/rouge/lexers/opentype_feature_file.rb

@@ -91,7 +91,6 @@ module Rouge
       state :strings do
         rule %r/"/, Str, :pop!
         rule %r/[^"%\n]+/, Str
-        rule %r/(\([a-z0-9_]+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?/i, Str
       end
     end
   end

data/lib/rouge/lexers/perl.rb

@@ -110,9 +110,10 @@ module Rouge
 
         rule %r/(__(END|DATA)__)\b/, Comment::Preproc, :end_part
         rule %r/\$\^[ADEFHILMOPSTWX]/, Name::Variable::Global
-        rule %r/\$[\\"'\[\]&`+*.,;=%~?@$!<>(^\|\/-](?!\w)/, Name::Variable::Global
+        rule %r/\$[\\"'\[\]&`+*.,;=%~?@$!<>(^\|\/_-](?!\w)/, Name::Variable::Global
+        rule %r/[$@%&*][$@%&*#_]*(?=[a-z{\[;])/i, Name::Variable, :varname
+
         rule %r/[-+\/*%=<>&^\|!\\~]=?/, Operator
-        rule %r/[$@%#]+/, Name::Variable, :varname
 
         rule %r/0_?[0-7]+(_[0-7]+)*/, Num::Oct
         rule %r/0x[0-9A-Fa-f]+(_[0-9A-Fa-f]+)*/, Num::Hex
@@ -159,8 +160,9 @@ module Rouge
 
       state :varname do
         rule %r/\s+/, Text
-        rule %r/\{/, Punctuation, :pop! # hash syntax
-        rule %r/\)|,/, Punctuation, :pop! # arg specifier
+        rule %r/[{\[]/, Punctuation, :pop! # hash syntax
+        rule %r/[),]/, Punctuation, :pop! # arg specifier
+        rule %r/[;]/, Punctuation, :pop! # postfix
         mixin :name_common
       end
 

data/lib/rouge/lexers/powershell.rb

@@ -131,34 +131,50 @@ module Rouge
         rule %r/[:,]/, Punctuation
       end
 
-      state :hasht do
-        rule %r/\s+/, Text::Whitespace
-        rule %r/\}/, Punctuation, :pop!
+      state :expr do
+        mixin :comments
         rule %r/"/, Str::Double, :dq
         rule %r/'/, Str::Single, :sq
-        rule %r/\w+/, Name::Other
+        rule %r/@"/, Str::Heredoc, :heredoc
+        rule %r/@'.*?'@/m, Str::Heredoc
+        rule %r/\d*\.\d+/, Num::Float
+        rule %r/\d+/, Num::Integer
+        rule %r/@\{/, Punctuation, :hasht
+        rule %r/@\(/, Punctuation, :array
+        rule %r/{/, Punctuation, :brace
+        rule %r/\[/, Punctuation, :bracket
+      end
+
+      state :hasht do
+        rule %r/\}/, Punctuation, :pop!
         rule %r/=/, Operator
-        rule %r/,/, Punctuation
+        rule %r/[,;]/, Punctuation
+        mixin :expr
+        rule %r/\w+/, Name::Other
         mixin :variable
       end
 
       state :array do
         rule %r/\s+/, Text::Whitespace
         rule %r/\)/, Punctuation, :pop!
-        rule %r/"/, Str::Double, :dq
-        rule %r/'/, Str::Single, :sq
-        rule %r/,/, Punctuation
+        rule %r/[,;]/, Punctuation
+        mixin :expr
         mixin :variable
       end
 
+      state :brace do
+        rule %r/[}]/, Punctuation, :pop!
+        mixin :root
+      end
+
       state :bracket do
         rule %r/\]/, Punctuation, :pop!
-        rule %r/[A-Za-z]\w+\./, Name::Constant
+        rule %r/[A-Za-z]\w+\./, Name
         rule %r/([A-Za-z]\w+)/ do |m|
           if ATTRIBUTES.include? m[0]
             token Name::Builtin::Pseudo
           else
-            token Keyword::Type
+            token Name
           end
         end
         mixin :root
@@ -174,20 +190,15 @@ module Rouge
         mixin :root
       end
 
-      state :root do
+      state :comments do
         rule %r/\s+/, Text::Whitespace
-
-        rule %r/#requires\s-version \d(?:\.\d+)?/, Comment::Preproc
         rule %r/#.*/, Comment
         rule %r/<#/, Comment::Multiline, :multiline
+      end
 
-        rule %r/"/, Str::Double, :dq
-        rule %r/'/, Str::Single, :sq
-        rule %r/@"/, Str::Heredoc, :heredoc
-        rule %r/@'.*?'@/m, Str::Heredoc
-
-        rule %r/\d*\.\d+/, Num::Float
-        rule %r/\d+/, Num::Integer
+      state :root do
+        mixin :comments
+        rule %r/#requires\s-version \d(?:\.\d+)?/, Comment::Preproc
 
         rule %r/\.\.(?=\.?\d)/, Operator
         rule %r/(?:#{OPERATORS})\b/i, Operator
@@ -204,28 +215,27 @@ module Rouge
         rule %r/-{1,2}\w+/, Name::Tag
 
         rule %r/(\.)?([-\w]+)(\[)/ do |m|
-          groups Operator, Name::Function, Punctuation
+          groups Operator, Name, Punctuation
           push :bracket
         end
 
-        rule %r/([\/\\~\w][-.:\/\\~\w]*)(\n)?/ do |m|
-          groups Name::Function, Text::Whitespace
+        rule %r/([\/\\~[a-z]][-.:\/\\~\w]*)(\n)?/i do |m|
+          groups Name, Text::Whitespace
           push :parameters
         end
 
-        rule %r/(\.)?([-\w]+)(?:(\()|(\n))?/ do |m|
+        rule %r/(\.)([-\w]+)(?:(\()|(\n))?/ do |m|
           groups Operator, Name::Function, Punctuation, Text::Whitespace
           push :parameters unless m[3].nil?
         end
 
         rule %r/\?/, Name::Function, :parameters
-        rule %r/[-+*\/%=!.&|]/, Operator
-        rule %r/@\{/, Punctuation, :hasht
-        rule %r/@\(/, Punctuation, :array
-        rule %r/\[/, Punctuation, :bracket
-        rule %r/[{}(),:;]/, Punctuation
 
+        mixin :expr
         mixin :variable
+
+        rule %r/[-+*\/%=!.&|]/, Operator
+        rule %r/[{}(),:;]/, Punctuation
       end
     end
   end

data/lib/rouge/lexers/q.rb

@@ -118,7 +118,7 @@ module Rouge
       end
 
       state :bottom do
-        rule %r/.*\z/m, Comment::Multiline
+        rule %r/.+\z/m, Comment::Multiline
       end
     end
   end

data/lib/rouge/lexers/rust.rb

@@ -22,8 +22,8 @@ module Rouge
 
       def self.keywords
         @keywords ||= %w(
-          as assert async await break const copy do drop else enum extern fail false
-          fn for if impl let log loop match mod move mut priv pub pure
+          as assert async await break const continue copy do drop else enum extern
+          fail false fn for if impl let log loop match mod move mut priv pub pure
           ref return self static struct true trait type unsafe use where
           while box
         )
@@ -99,6 +99,7 @@ module Rouge
         rule %r/[*\/!@~&+%^<>=\?-]|\.{2,3}/, Operator
 
         rule %r/([.]\s*)?#{id}(?=\s*[(])/m, Name::Function
+        rule %r/[.]\s*await\b/, Keyword
         rule %r/[.]\s*#{id}/, Name::Property
         rule %r/[.]\s*\d+/, Name::Attribute
         rule %r/(#{id})(::)/m do

data/lib/rouge/lexers/smarty.rb

@@ -41,7 +41,7 @@ module Rouge
         end
 
 
-        rule(/.*?(?={[\/a-zA-Z0-9$#*"'])|.*/m) { delegate parent }
+        rule(/.+?(?={[\/a-zA-Z0-9$#*"'])/m) { delegate parent }
         rule(/.+/m) { delegate parent }
       end
 

data/lib/rouge/lexers/velocity.rb

@@ -63,7 +63,7 @@ module Rouge
         rule %r/0[xX][0-9a-fA-F]+[Ll]?/, Num::Hex
         rule %r/\b[0-9]+\b/, Num::Integer
         rule %r/(true|false|null)\b/, Keyword::Constant
-        rule %r/[(\[]/, Punctuation, :push!
+        rule %r/[(\[]/, Punctuation, :push
         rule %r/[)\]}]/, Punctuation, :pop!
       end
     end

data/lib/rouge/lexers/wollok.rb

@@ -18,7 +18,6 @@ module Rouge
 
       state :whitespaces_and_comments do
         rule %r/\s+/m, Text::Whitespace
-        rule %r/$+/m, Text::Whitespace
         rule %r(//.*$), Comment::Single
         rule %r(/\*(.|\s)*?\*/)m, Comment::Multiline
       end

data/lib/rouge/regex_lexer.rb

@@ -6,6 +6,37 @@ module Rouge
   # A stateful lexer that uses sets of regular expressions to
   # tokenize a string.  Most lexers are instances of RegexLexer.
   class RegexLexer < Lexer
+    class InvalidRegex < StandardError
+      def initialize(re)
+        @re = re
+      end
+
+      def to_s
+        "regex #{@re.inspect} matches empty string, but has no predicate!"
+      end
+    end
+
+    class ClosedState < StandardError
+      attr_reader :state
+      def initialize(state)
+        @state = state
+      end
+
+      def rule
+        @state.rules.last
+      end
+
+      def to_s
+        rule = @state.rules.last
+        msg = "State :#{state.name} cannot continue after #{rule.inspect}, which will always match."
+        if rule.re.source.include?('*')
+          msg += " Consider replacing * with +."
+        end
+
+        msg
+      end
+    end
+
     # A rule is a tuple of a regular expression to test, and a callback
     # to perform if the test succeeds.
     #
@@ -42,12 +73,13 @@ module Rouge
     end
 
     class StateDSL
-      attr_reader :rules
+      attr_reader :rules, :name
       def initialize(name, &defn)
         @name = name
         @defn = defn
         @rules = []
         @loaded = false
+        @closed = false
       end
 
       def to_state(lexer_class)
@@ -95,10 +127,14 @@ module Rouge
       #   {RegexLexer#token}, and {RegexLexer#delegate}.  The first
       #   argument can be used to access the match groups.
       def rule(re, tok=nil, next_state=nil, &callback)
+        raise ClosedState.new(self) if @closed
+
         if tok.nil? && callback.nil?
           raise "please pass `rule` a token to yield or a callback"
         end
 
+        matches_empty = re =~ ''
+
         callback ||= case next_state
         when :pop!
           proc do |stream|
@@ -123,6 +159,9 @@ module Rouge
             @stack.push(state)
           end
         when nil
+          # cannot use an empty-matching regexp with no predicate
+          raise InvalidRegex.new(re) if matches_empty
+
           proc do |stream|
             puts "    yielding: #{tok.qualname}, #{stream[0].inspect}" if @debug
             @output_stream.call(tok, stream[0])
@@ -132,6 +171,22 @@ module Rouge
         end
 
         rules << Rule.new(re, callback)
+
+        close! if matches_empty && !context_sensitive?(re)
+      end
+
+      def context_sensitive?(re)
+        source = re.source
+        return true if source =~ /[(][?]<?[!=]/
+
+        # anchors count as lookahead/behind
+        return true if source =~ /[$^]/
+
+        false
+      end
+
+      def close!
+        @closed = true
       end
 
       # Mix in the rules from another state into this state.  The rules

data/lib/rouge/version.rb

@@ -3,6 +3,6 @@
 
 module Rouge
   def self.version
-    "3.23.0"
+    "3.25.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rouge
 version: !ruby/object:Gem::Version
-  version: 3.23.0
+  version: 3.25.0
 platform: ruby
 authors:
 - Jeanine Adkisson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-09-08 00:00:00.000000000 Z
+date: 2020-11-10 00:00:00.000000000 Z
 dependencies: []
 description: Rouge aims to a be a simple, easy-to-extend drop-in replacement for pygments.
 email:
@@ -73,6 +73,7 @@ files:
 - lib/rouge/demos/eiffel
 - lib/rouge/demos/elixir
 - lib/rouge/demos/elm
+- lib/rouge/demos/email
 - lib/rouge/demos/epp
 - lib/rouge/demos/erb
 - lib/rouge/demos/erlang
@@ -109,6 +110,7 @@ files:
 - lib/rouge/demos/irb
 - lib/rouge/demos/irb_output
 - lib/rouge/demos/isbl
+- lib/rouge/demos/j
 - lib/rouge/demos/janet
 - lib/rouge/demos/java
 - lib/rouge/demos/javascript
@@ -228,6 +230,7 @@ files:
 - lib/rouge/formatters/html.rb
 - lib/rouge/formatters/html_inline.rb
 - lib/rouge/formatters/html_legacy.rb
+- lib/rouge/formatters/html_line_highlighter.rb
 - lib/rouge/formatters/html_line_table.rb
 - lib/rouge/formatters/html_linewise.rb
 - lib/rouge/formatters/html_pygments.rb
@@ -296,6 +299,7 @@ files:
 - lib/rouge/lexers/eiffel.rb
 - lib/rouge/lexers/elixir.rb
 - lib/rouge/lexers/elm.rb
+- lib/rouge/lexers/email.rb
 - lib/rouge/lexers/epp.rb
 - lib/rouge/lexers/erb.rb
 - lib/rouge/lexers/erlang.rb
@@ -333,6 +337,7 @@ files:
 - lib/rouge/lexers/irb.rb
 - lib/rouge/lexers/isbl.rb
 - lib/rouge/lexers/isbl/builtins.rb
+- lib/rouge/lexers/j.rb
 - lib/rouge/lexers/janet.rb
 - lib/rouge/lexers/java.rb
 - lib/rouge/lexers/javascript.rb