rouge 3.21.0 → 3.30.0

data/lib/rouge/lexers/syzlang.rb

@@ -0,0 +1,317 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    class Syzlang < RegexLexer
+      title "Syzlang"
+      desc "Syscall description language used by syzkaller"
+      tag 'syzlang'
+
+      def self.keywords
+        @keywords ||= Set.new %w(
+          align breaks_returns dec define disabled hex ignore_return in incdir
+          include inet inout oct opt out packed parent prog_timeout pseudo
+          resource size syscall timeout type varlen
+        )
+      end
+
+      def self.keywords_type
+        @keywords_type ||= Set.new %w(
+          array bitsize bool16 bool32 bool64 bool8 boolptr buffer bytesize
+          bytesize2 bytesize4 bytesize8 const csum filename fileoff flags fmt
+          int16 int16be int32 int32be int64 int64be int8 int8be intptr len
+          offsetof optional proc ptr ptr64 string stringnoz text vma vma64 void
+        )
+      end
+
+      comment = /#.*$/
+      inline_spaces = /[ \t]+/
+      eol_spaces = /[\n\r]+/
+      spaces = /\s+/
+
+      state :inline_break do
+        rule inline_spaces, Text
+        rule %r//, Text, :pop!
+      end
+
+      state :space_break do
+        rule spaces, Text
+        rule comment, Comment
+        rule %r//, Text, :pop!
+      end
+
+      id = /[a-zA-Z_][a-zA-Z0-9_]*/
+      num_id = /[a-zA-Z0-9_]+/
+
+      state :mixin_name do
+        rule id, Name
+      end
+
+      state :mixin_number do
+        rule %r/-?0x[\da-f]+/i, Num::Hex
+        rule %r/-?\d+/, Num::Integer
+        rule %r/'[^']?'/, Str::Char
+      end
+
+      state :mixin_string do
+        rule %r/"[^"]*"/, Str::Double
+        rule %r/`[^`]*`/, Str::Backtick
+      end
+
+      state :mixin_term do
+        mixin :mixin_number
+        mixin :mixin_string
+
+        # Keywords.
+        rule id do |m|
+          if self.class.keywords.include?(m[0])
+            token Keyword
+          elsif self.class.keywords_type.include?(m[0])
+            token Keyword::Type
+          else
+            token Name
+          end
+        end
+
+        # Ranges.
+        rule %r/:/, Punctuation
+
+        # "struct$type" struct name format.
+        rule %r/\$/, Name
+      end
+
+      state :term_list do
+        rule spaces, Text
+        rule comment, Comment
+        mixin :mixin_term
+        rule %r/\[/, Punctuation, :term_list
+        rule %r/,/, Punctuation
+        rule %r/[\]\)]/, Punctuation, :pop!
+      end
+
+      state :arg_type do
+        mixin :mixin_term
+        rule %r/\[/, Punctuation, :term_list
+        rule %r//, Text, :pop!
+      end
+
+      state :include do
+        rule %r/(<)([^>]+)(>)/ do |m|
+          groups Punctuation, Str, Punctuation
+        end
+        rule %r//, Text, :pop!
+      end
+
+      state :define_name do
+        mixin :mixin_name
+        rule %r//, Text, :pop!
+      end
+
+      state :define_exp do
+        mixin :mixin_name
+        mixin :mixin_number
+        mixin :mixin_string
+        rule %r/[~!%\^&\*\-\+\/\|<>\?:]/, Operator
+        rule %r/[\(\){}\[\];,]/, Punctuation
+        rule inline_spaces, Text
+        rule %r//, Text, :pop!
+      end
+
+      state :resource_name do
+        mixin :mixin_name
+        rule %r//, Text, :pop!
+      end
+
+      state :resource_type do
+        rule %r/\[/, Punctuation, :arg_type
+        rule %r/\]/, Punctuation, :pop!
+      end
+
+      state :resource_values do
+        rule %r/:/ do
+          token Punctuation
+          push :resource_values_list
+          push :space_break
+        end
+        rule %r//, Text, :pop!
+      end
+
+      state :resource_values_list do
+        rule inline_spaces, Text
+        mixin :mixin_name
+        mixin :mixin_number
+        mixin :mixin_string
+        rule %r/,/, Punctuation, :space_break
+        rule %r//, Text, :pop!
+      end
+
+      state :flags_list do
+        rule inline_spaces, Text
+        rule %r/\./, Punctuation
+        mixin :mixin_name
+        mixin :mixin_number
+        mixin :mixin_string
+        rule %r/,/, Punctuation, :space_break
+        rule %r//, Punctuation, :pop!
+      end
+
+      state :syscall_args do
+        rule spaces, Text
+        rule comment, Comment
+        rule %r/\./, Punctuation
+        rule id do
+          token Name
+          push :arg_type
+          push :space_break
+        end
+        rule %r/,/, Punctuation
+        rule %r/\)/, Punctuation, :pop!
+      end
+
+      state :syscall_retval do
+        mixin :mixin_name
+        rule %r//, Text, :pop!
+      end
+
+      state :syscall_mods do
+        rule %r/\(/, Punctuation, :term_list
+        rule %r//, Text, :pop!
+      end
+
+      state :struct_fields do
+        rule id do
+          token Name
+          push :space_break
+          push :struct_field_mods
+          push :inline_break
+          push :arg_type
+          push :space_break
+        end
+        rule %r/[}\]]/, Punctuation, :pop!
+      end
+
+      state :struct_field_mods do
+        rule %r/\(/, Punctuation, :term_list
+        rule %r//, Text, :pop!
+      end
+
+      state :struct_mods do
+        rule %r/\[/, Punctuation, :term_list
+        rule %r//, Text, :pop!
+      end
+
+      state :type_name do
+        mixin :mixin_name
+        rule %r//, Text, :pop!
+      end
+
+      state :type_args do
+        rule %r/\[/, Punctuation, :type_args_list
+        rule %r//, Text, :pop!
+      end
+
+      state :type_args_list do
+        rule spaces, Text
+        rule comment, Comment
+        mixin :mixin_name
+        rule %r/,/, Punctuation
+        rule %r/\]/, Punctuation, :pop!
+      end
+
+      state :type_body do
+        rule %r/[{\[]/ do
+          token Punctuation
+          pop!
+          push :space_break
+          push :struct_mods
+          push :inline_break
+          push :struct_fields
+          push :space_break
+        end
+        rule %r// do
+          pop!
+          push :arg_type
+        end
+      end
+
+      state :root do
+        # Whitespace.
+        rule spaces, Text
+
+        # Comments.
+        rule comment, Comment
+
+        # Includes.
+        rule %r/(include|incdir)/ do
+          token Keyword
+          push :include
+          push :space_break
+        end
+
+        # Defines.
+        rule %r/define/ do
+          token Keyword
+          push :define_exp
+          push :space_break
+          push :define_name
+          push :space_break
+        end
+
+        # Resources.
+        rule %r/resource/ do
+          token Keyword
+          push :resource_values
+          push :inline_break
+          push :resource_type
+          push :inline_break
+          push :resource_name
+          push :space_break
+        end
+
+        # Flags and strings.
+        rule %r/(#{id}|_)(#{spaces})(=)/ do |m|
+          if m[1] == "_"
+            groups Keyword, Text, Punctuation
+          else
+            groups Name, Text, Punctuation
+          end
+          push :flags_list
+          push :space_break
+        end
+
+        # Syscalls.
+        rule %r/(#{id})(\$)?(#{num_id})?(#{spaces})?(\()/ do |m|
+          groups Name::Function, Punctuation, Name::Function::Magic, Text, Punctuation
+          push :syscall_mods
+          push :inline_break
+          push :syscall_retval
+          push :inline_break
+          push :syscall_args
+          push :space_break
+        end
+
+        # Structs and unions.
+        rule %r/(#{id}|#{id}\$#{num_id})(#{spaces})?([{\[])/ do |m|
+          groups Name, Text, Punctuation
+          push :inline_break
+          push :struct_mods
+          push :inline_break
+          push :struct_fields
+          push :space_break
+        end
+
+        # Types.
+        rule %r/type/ do
+          token Keyword
+          push :type_body
+          push :space_break
+          push :type_args
+          push :type_name
+          push :space_break
+        end
+      end
+
+    end
+  end
+end

data/lib/rouge/lexers/syzprog.rb

@@ -0,0 +1,122 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    class Syzprog < RegexLexer
+      title "Syzprog"
+      desc "Program description language used by syzkaller"
+      tag 'syzprog'
+
+      def self.keywords
+        @keywords ||= Set.new %w(
+          ANY ANYBLOB ANYPTR ANYPTR64 ANYPTRS ANYRES16 ANYRES32 ANYRES64
+          ANYRESDEC ANYRESHEX ANYRESOCT ANYUNION AUTO false nil true void
+          async fail_nth rerun
+        )
+      end
+
+      comment = /#.*$/
+      inline_spaces = /[ \t]+/
+      eol_spaces = /[\n\r]+/
+      spaces = /\s+/
+
+      id = /[a-zA-Z_][a-zA-Z0-9_]*/
+      num_id = /[a-zA-Z0-9_]+/
+      res_id = /r[0-9]+/
+
+      state :inline_break do
+        rule inline_spaces, Text
+        rule %r//, Text, :pop!
+      end
+
+      state :eol_break do
+        rule eol_spaces, Text
+        rule comment, Comment
+        rule %r//, Text, :pop!
+      end
+
+      state :space_break do
+        rule spaces, Text
+        rule comment, Comment
+        rule %r//, Text, :pop!
+      end
+
+      state :mixin_number do
+        rule %r/-?0x[\da-f]+/i, Num::Hex
+        rule %r/-?\d+/, Num::Integer
+      end
+
+      state :mixin_string do
+        rule %r/"[^"]*"/, Str::Double
+        rule %r/`[^`]*`/, Str::Backtick
+        rule %r/'[^']*'/, Str::Single
+      end
+
+      state :mixin_term do
+        mixin :mixin_number
+        mixin :mixin_string
+
+        rule %r/#{res_id}/, Keyword::Pseudo
+        rule id do |m|
+          if self.class.keywords.include?(m[0])
+            token Keyword
+          else
+            token Name
+          end
+        end
+      end
+
+      state :mods_list do
+        rule spaces, Text
+        rule comment, Comment
+        mixin :mixin_term
+        rule %r/[,:]/, Punctuation
+        rule %r/\)/, Punctuation, :pop!
+      end
+
+      state :syscall_mods do
+        rule %r/\(/, Punctuation, :mods_list
+        rule %r//, Text, :pop!
+      end
+
+      state :syscall_args do
+        rule spaces, Text
+        rule comment, Comment
+        mixin :mixin_term
+        mixin :mixin_number
+        mixin :mixin_string
+        # This punctuation is a part of the syntax:
+        rule %r/[@&=,<>{}\[\]]/, Punctuation
+        # This punctuation is not, highlight just in case:
+        rule %r/[!#\$%\^\*\-\+\/\|~:;.\?]/, Punctuation
+        rule %r/\(/, Punctuation, :syscall_args
+        rule %r/\)/, Punctuation, :pop!
+      end
+
+      state :root do
+        # Whitespace.
+        rule spaces, Text
+
+        # Comments.
+        rule comment, Comment
+
+        # Return values.
+        rule %r/(#{res_id})(#{spaces})(=)/ do
+          groups Keyword::Pseudo, Text, Punctuation
+        end
+
+        # Syscalls.
+        rule %r/(#{id})(\$)?(#{num_id})?(#{spaces})?(\()/ do |m|
+          groups Name::Function, Punctuation, Name::Function::Magic, Text, Punctuation
+          push :syscall_mods
+          push :inline_break
+          push :syscall_args
+          push :space_break
+        end
+
+      end
+
+    end
+  end
+end

data/lib/rouge/lexers/tap.rb

@@ -86,4 +86,3 @@ module Rouge
     end
   end
 end
-

data/lib/rouge/lexers/toml.rb

@@ -11,14 +11,15 @@ module Rouge
       filenames '*.toml', 'Pipfile'
       mimetypes 'text/x-toml'
 
-      identifier = /\S+/
+      # bare keys and quoted keys
+      identifier = %r/(?:\S+|"[^"]+"|'[^']+')/
 
       state :basic do
         rule %r/\s+/, Text
         rule %r/#.*?$/, Comment
         rule %r/(true|false)/, Keyword::Constant
 
-        rule %r/(\S+)(\s*)(=)(\s*)(\{)/ do |m|
+        rule %r/(#{identifier})(\s*)(=)(\s*)(\{)/ do
           groups Name::Namespace, Text, Operator, Text, Punctuation
           push :inline
         end
@@ -27,9 +28,14 @@ module Rouge
 
         rule %r/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z/, Literal::Date
 
-        rule %r/(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?j?/, Num::Float
-        rule %r/\d+[eE][+-]?[0-9]+j?/, Num::Float
-        rule %r/\-?\d+/, Num::Integer
+        rule %r/[+-]?\d+(?:_\d+)*\.\d+(?:_\d+)*(?:[eE][+-]?\d+(?:_\d+)*)?/, Num::Float
+        rule %r/[+-]?\d+(?:_\d+)*[eE][+-]?\d+(?:_\d+)*/, Num::Float
+        rule %r/[+-]?(?:nan|inf)/, Num::Float
+
+        rule %r/0x\h+(?:_\h+)*/, Num::Hex
+        rule %r/0o[0-7]+(?:_[0-7]+)*/, Num::Oct
+        rule %r/0b[01]+(?:_[01]+)*/, Num::Bin
+        rule %r/[+-]?\d+(?:_\d+)*/, Num::Integer
       end
 
       state :root do
@@ -48,6 +54,11 @@ module Rouge
 
       state :content do
         mixin :basic
+
+        rule %r/(#{identifier})(\s*)(=)/ do
+          groups Name::Property, Text, Punctuation
+        end
+
         rule %r/"""/, Str, :mdq
         rule %r/"/, Str, :dq
         rule %r/'''/, Str, :msq
@@ -95,10 +106,6 @@ module Rouge
       state :inline do
         mixin :content
 
-        rule %r/(#{identifier})(\s*)(=)/ do
-          groups Name::Property, Text, Punctuation
-        end
-
         rule %r/\}/, Punctuation, :pop!
       end
     end

data/lib/rouge/lexers/tsx.rb

@@ -24,4 +24,3 @@ module Rouge
     end
   end
 end
-

data/lib/rouge/lexers/tulip.rb

@@ -105,4 +105,3 @@ module Rouge
     end
   end
 end
-

data/lib/rouge/lexers/velocity.rb

@@ -63,7 +63,7 @@ module Rouge
         rule %r/0[xX][0-9a-fA-F]+[Ll]?/, Num::Hex
         rule %r/\b[0-9]+\b/, Num::Integer
         rule %r/(true|false|null)\b/, Keyword::Constant
-        rule %r/[(\[]/, Punctuation, :push!
+        rule %r/[(\[]/, Punctuation, :push
         rule %r/[)\]}]/, Punctuation, :pop!
       end
     end

data/lib/rouge/lexers/viml/keywords.rb

@@ -18,4 +18,4 @@ module Rouge
       end
     end
   end
-end
+end

data/lib/rouge/lexers/wollok.rb

@@ -18,7 +18,6 @@ module Rouge
 
       state :whitespaces_and_comments do
         rule %r/\s+/m, Text::Whitespace
-        rule %r/$+/m, Text::Whitespace
         rule %r(//.*$), Comment::Single
         rule %r(/\*(.|\s)*?\*/)m, Comment::Multiline
       end

data/lib/rouge/lexers/yaml.rb

@@ -175,7 +175,7 @@ module Rouge
 
       state :block_nodes do
         # implicit key
-        rule %r/((?:[\p{L}\p{Nl}\p{Nd}_][\p{L}\p{Nl}\p{Nd}\p{Blank}_-]*)?)(:)(?=\s|$)/ do |m|
+        rule %r/([^#,:?\[\]{}"'\n]+)(:)(?=\s|$)/ do |m|
           groups Name::Attribute, Punctuation::Indicator
           set_indent m[0], :implicit => true
         end

data/lib/rouge/regex_lexer.rb

@@ -6,6 +6,37 @@ module Rouge
   # A stateful lexer that uses sets of regular expressions to
   # tokenize a string.  Most lexers are instances of RegexLexer.
   class RegexLexer < Lexer
+    class InvalidRegex < StandardError
+      def initialize(re)
+        @re = re
+      end
+
+      def to_s
+        "regex #{@re.inspect} matches empty string, but has no predicate!"
+      end
+    end
+
+    class ClosedState < StandardError
+      attr_reader :state
+      def initialize(state)
+        @state = state
+      end
+
+      def rule
+        @state.rules.last
+      end
+
+      def to_s
+        rule = @state.rules.last
+        msg = "State :#{state.name} cannot continue after #{rule.inspect}, which will always match."
+        if rule.re.source.include?('*')
+          msg += " Consider replacing * with +."
+        end
+
+        msg
+      end
+    end
+
     # A rule is a tuple of a regular expression to test, and a callback
     # to perform if the test succeeds.
     #
@@ -42,12 +73,13 @@ module Rouge
     end
 
     class StateDSL
-      attr_reader :rules
+      attr_reader :rules, :name
       def initialize(name, &defn)
         @name = name
         @defn = defn
         @rules = []
         @loaded = false
+        @closed = false
       end
 
       def to_state(lexer_class)
@@ -95,10 +127,14 @@ module Rouge
       #   {RegexLexer#token}, and {RegexLexer#delegate}.  The first
       #   argument can be used to access the match groups.
       def rule(re, tok=nil, next_state=nil, &callback)
+        raise ClosedState.new(self) if @closed
+
         if tok.nil? && callback.nil?
           raise "please pass `rule` a token to yield or a callback"
         end
 
+        matches_empty = re =~ ''
+
         callback ||= case next_state
         when :pop!
           proc do |stream|
@@ -123,6 +159,9 @@ module Rouge
             @stack.push(state)
           end
         when nil
+          # cannot use an empty-matching regexp with no predicate
+          raise InvalidRegex.new(re) if matches_empty
+
           proc do |stream|
             puts "    yielding: #{tok.qualname}, #{stream[0].inspect}" if @debug
             @output_stream.call(tok, stream[0])
@@ -132,6 +171,22 @@ module Rouge
         end
 
         rules << Rule.new(re, callback)
+
+        close! if matches_empty && !context_sensitive?(re)
+      end
+
+      def context_sensitive?(re)
+        source = re.source
+        return true if source =~ /[(][?]<?[!=]/
+
+        # anchors count as lookahead/behind
+        return true if source =~ /[$^]/
+
+        false
+      end
+
+      def close!
+        @closed = true
       end
 
       # Mix in the rules from another state into this state.  The rules

data/lib/rouge/themes/base16.rb

@@ -67,6 +67,7 @@ module Rouge
       style Keyword::Declaration, :fg => :base09
 
       style Literal::String, :fg => :base0B
+      style Literal::String::Affix, :fg => :base0E
       style Literal::String::Regex, :fg => :base0C
 
       style Literal::String::Interpol,

data/lib/rouge/themes/bw.rb

@@ -26,6 +26,7 @@ module Rouge
       style Name::Tag,                   :bold => true
 
       style Literal::String,             :italic => true
+      style Literal::String::Affix,      :bold => true
       style Literal::String::Interpol,   :bold => true
       style Literal::String::Escape,     :bold => true
 

data/lib/rouge/themes/colorful.rb

@@ -37,6 +37,7 @@ module Rouge
       style Name::Decorator,             :fg => "#555", :bold => true
 
       style Literal::String,             :bg => "#fff0f0"
+      style Literal::String::Affix,      :fg => "#080", :bold => true
       style Literal::String::Char,       :fg => "#04D"
       style Literal::String::Doc,        :fg => "#D42"
       style Literal::String::Interpol,   :bg => "#eee"

data/lib/rouge/themes/github.rb

@@ -35,6 +35,7 @@ module Rouge
       style Literal::Number::Integer,         :fg => '#009999'
       style Literal::Number::Oct,             :fg => '#009999'
       style Literal::Number,                  :fg => '#009999'
+      style Literal::String::Affix,           :fg => '#000000', :bold => true
       style Literal::String::Backtick,        :fg => '#d14'
       style Literal::String::Char,            :fg => '#d14'
       style Literal::String::Doc,             :fg => '#d14'

data/lib/rouge/themes/gruvbox.rb

@@ -150,6 +150,8 @@ module Rouge
             Literal::String::Interpol,
             Literal::String::Regex, :fg => :green, :italic => true
 
+      style Literal::String::Affix, :fg => :red
+
       style Literal::String::Escape, :fg => :orange
 
       style Name::Namespace,