rouge 3.2.0 → 3.18.0

data/lib/rouge/lexers/elixir.rb

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*- #
+# frozen_string_literal: true
 
 module Rouge
   module Lexers
@@ -16,66 +17,80 @@ module Rouge
       mimetypes 'text/x-elixir', 'application/x-elixir'
 
       state :root do
-        rule /\s+/m, Text
-        rule /#.*$/, Comment::Single
+        rule %r/\s+/m, Text
+        rule %r/#.*$/, Comment::Single
         rule %r{\b(case|cond|end|bc|lc|if|unless|try|loop|receive|fn|defmodule|
              defp?|defprotocol|defimpl|defrecord|defmacrop?|defdelegate|
-             defexception|exit|raise|throw|after|rescue|catch|else)\b(?![?!])|
+             defexception|defguardp?|defstruct|exit|raise|throw|after|rescue|catch|else)\b(?![?!])|
              (?<!\.)\b(do|\-\>)\b}x, Keyword
-        rule /\b(import|require|use|recur|quote|unquote|super|refer)\b(?![?!])/, Keyword::Namespace
-        rule /(?<!\.)\b(and|not|or|when|xor|in)\b/, Operator::Word
+        rule %r/\b(import|require|use|recur|quote|unquote|super|refer)\b(?![?!])/, Keyword::Namespace
+        rule %r/(?<!\.)\b(and|not|or|when|xor|in)\b/, Operator::Word
         rule %r{%=|\*=|\*\*=|\+=|\-=|\^=|\|\|=|
-             <=>|<(?!<|=)|>(?!<|=|>)|<=|>=|===|==|=~|!=|!~|(?=[\s\t])\?|
-             (?<=[\s\t])!+|&(&&?|(?!\d))|\|\||\^|\*|\+|\-|/|
+             <=>|<(?!<|=)|>(?!<|=|>)|<=|>=|===|==|=~|!=|!~|(?=[\s])\?|
+             (?<=[\s])!+|&(&&?|(?!\d))|\|\||\^|\*|\+|\-|/|
              \||\+\+|\-\-|\*\*|\/\/|\<\-|\<\>|<<|>>|=|\.|~~~}x, Operator
         rule %r{(?<!:)(:)([a-zA-Z_]\w*([?!]|=(?![>=]))?|\<\>|===?|>=?|<=?|
              <=>|&&?|%\(\)|%\[\]|%\{\}|\+\+?|\-\-?|\|\|?|\!|//|[%&`/\|]|
              \*\*?|=?~|<\-)|([a-zA-Z_]\w*([?!])?)(:)(?!:)}, Str::Symbol
-        rule /:"/, Str::Symbol, :interpoling_symbol
-        rule /\b(nil|true|false)\b(?![?!])|\b[A-Z]\w*\b/, Name::Constant
-        rule /\b(__(FILE|LINE|MODULE|MAIN|FUNCTION)__)\b(?![?!])/, Name::Builtin::Pseudo
-        rule /[a-zA-Z_!][\w_]*[!\?]?/, Name
+        rule %r/:"/, Str::Symbol, :interpoling_symbol
+        rule %r/\b(nil|true|false)\b(?![?!])|\b[A-Z]\w*\b/, Name::Constant
+        rule %r/\b(__(FILE|LINE|MODULE|MAIN|FUNCTION)__)\b(?![?!])/, Name::Builtin::Pseudo
+        rule %r/[a-zA-Z_!]\w*[!\?]?/, Name
         rule %r{::|[%(){};,/\|:\\\[\]]}, Punctuation
-        rule /@[a-zA-Z_]\w*|&\d/, Name::Variable
-        rule %r{\b(0[xX][0-9A-Fa-f]+|\d(_?\d)*(\.(?![^\d\s])
-             (_?\d)*)?([eE][-+]?\d(_?\d)*)?|0[bB][01]+)\b}x, Num
+        rule %r/@[a-zA-Z_]\w*|&\d/, Name::Variable
+        rule %r{\b\d(_?\d)*(\.(?![^\d\s])(_?\d)+)([eE][-+]?\d(_?\d)*)?\b}, Num::Float
+        rule %r{\b0x[0-9A-Fa-f](_?[0-9A-Fa-f])*\b}, Num::Hex
+        rule %r{\b0o[0-7](_?[0-7])*\b}, Num::Oct
+        rule %r{\b0b[01](_?[01])*\b}, Num::Bin
+        rule %r{\b\d(_?\d)*\b}, Num::Integer
 
         mixin :strings
         mixin :sigil_strings
       end
 
       state :strings do
-        rule /(%[A-Ba-z])?"""(?:.|\n)*?"""/, Str::Doc
-        rule /'''(?:.|\n)*?'''/, Str::Doc
-        rule /"/, Str::Doc, :dqs
-        rule /'.*?'/, Str::Single
+        rule %r/(%[A-Ba-z])?"""(?:.|\n)*?"""/, Str::Doc
+        rule %r/'''(?:.|\n)*?'''/, Str::Doc
+        rule %r/"/, Str::Double, :dqs
+        rule %r/'/, Str::Single, :sqs
         rule %r{(?<!\w)\?(\\(x\d{1,2}|\h{1,2}(?!\h)\b|0[0-7]{0,2}(?![0-7])\b[^x0MC])|(\\[MC]-)+\w|[^\s\\])}, Str::Other
-
       end
 
       state :dqs do
-        rule /"/, Str::Double, :pop!
-        mixin :enddoublestr
+        mixin :escapes
+        mixin :interpoling
+        rule %r/[^#"\\]+/, Str::Double
+        rule %r/"/, Str::Double, :pop!
+        rule %r/[#\\]/, Str::Double
+      end
+
+      state :sqs do
+        mixin :escapes
+        mixin :interpoling
+        rule %r/[^#'\\]+/, Str::Single
+        rule %r/'/, Str::Single, :pop!
+        rule %r/[#\\]/, Str::Single
       end
 
       state :interpoling do
-        rule /#\{/, Str::Interpol, :interpoling_string
+        rule %r/#\{/, Str::Interpol, :interpoling_string
       end
 
       state :interpoling_string do
-        rule /\}/, Str::Interpol, :pop!
+        rule %r/\}/, Str::Interpol, :pop!
         mixin :root
       end
 
-      state :interpoling_symbol do
-        rule /"/, Str::Symbol, :pop!
-        mixin :interpoling
-        rule /[^#"]+/, Str::Symbol
+      state :escapes do
+        rule %r/\\x\h{2}/, Str::Escape
+        rule %r/\\u\{?\d+\}?/, Str::Escape
+        rule %r/\\[\\abdefnrstv0"']/, Str::Escape
       end
 
-      state :enddoublestr do
+      state :interpoling_symbol do
+        rule %r/"/, Str::Symbol, :pop!
         mixin :interpoling
-        rule /[^#"]+/, Str::Double
+        rule %r/[^#"]+/, Str::Symbol
       end
 
       state :sigil_strings do
@@ -85,7 +100,7 @@ module Rouge
         delimiter_map = { '{' => '}', '[' => ']', '(' => ')', '<' => '>' }
         # Match a-z for custom sigils too
         sigil_opens = Regexp.union(delimiter_map.keys + %w(| / ' "))
-        rule /~([A-Za-z])?(#{sigil_opens})/ do |m|
+        rule %r/~([A-Za-z])?(#{sigil_opens})/ do |m|
           open = Regexp.escape(m[2])
           close = Regexp.escape(delimiter_map[m[2]] || m[2])
           interp = /[SRCW]/ === m[1]
@@ -107,26 +122,27 @@ module Rouge
           token toktype
 
           push do
-            rule /#{close}/, toktype, :pop!
+            rule %r/#{close}/, toktype, :pop!
 
             if interp
               mixin :interpoling
-              rule /#/, toktype
+              rule %r/#/, toktype
             else
-              rule /[\\#]/, toktype
+              rule %r/[\\#]/, toktype
             end
 
-            rule /[^##{open}#{close}\\]+/m, toktype
+            uniq_chars = "#{open}#{close}".squeeze
+            rule %r/[^##{uniq_chars}\\]+/m, toktype
           end
         end
       end
 
       state :regex_flags do
-        rule /[fgimrsux]*/, Str::Regex, :pop!
+        rule %r/[fgimrsux]*/, Str::Regex, :pop!
       end
 
       state :list_flags do
-        rule /[csa]?/, Str::Other, :pop!
+        rule %r/[csa]?/, Str::Other, :pop!
       end
     end
   end

data/lib/rouge/lexers/elm.rb

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*- #
+# frozen_string_literal: true
 
 module Rouge
   module Lexers
@@ -22,66 +23,66 @@ module Rouge
 
       state :root do
         # Whitespaces
-        rule /\s+/m, Text
+        rule %r/\s+/m, Text
         # Single line comments
-        rule /--.*/, Comment::Single
+        rule %r/--.*/, Comment::Single
         # Multiline comments
-        rule /{-/, Comment::Multiline, :multiline_comment
+        rule %r/{-/, Comment::Multiline, :multiline_comment
 
         # Keywords
-        rule /\b(#{keywords.join('|')})\b/, Keyword
+        rule %r/\b(#{keywords.join('|')})\b/, Keyword
 
         # Variable or a function
-        rule /[a-z][\w]*/, Name
+        rule %r/[a-z]\w*/, Name
         # Underscore is a name for a variable, when it won't be used later
-        rule /_/, Name
+        rule %r/_/, Name
         # Type
-        rule /[A-Z][\w]*/, Keyword::Type        
+        rule %r/[A-Z]\w*/, Keyword::Type
 
         # Two symbol operators: -> :: // .. && || ++ |> <| << >> == /= <= >=
-        rule /(->|::|\/\/|\.\.|&&|\|\||\+\+|\|>|<\||>>|<<|==|\/=|<=|>=)/, Operator
+        rule %r/(->|::|\/\/|\.\.|&&|\|\||\+\+|\|>|<\||>>|<<|==|\/=|<=|>=)/, Operator
         # One symbol operators: + - / * % = < > ^ | !
-        rule /[+-\/*%=<>^\|!]/, Operator
+        rule %r/[+-\/*%=<>^\|!]/, Operator
         # Lambda operator
-        rule /\\/, Operator
+        rule %r/\\/, Operator
         # Not standard Elm operators, but these symbols can be used for custom inflix operators. We need to highlight them as operators as well.
-        rule /[@\#$&~?]/, Operator
+        rule %r/[@\#$&~?]/, Operator
 
         # Single, double quotes, and triple double quotes
-        rule /"""/, Str, :multiline_string
-        rule /'(\\.|.)'/, Str::Char        
-        rule /"/, Str, :double_quote
+        rule %r/"""/, Str, :multiline_string
+        rule %r/'(\\.|.)'/, Str::Char
+        rule %r/"/, Str, :double_quote
 
         # Numbers
-        rule /0x[\da-f]+/i, Num::Hex
-        rule /\d+e[+-]?\d+/i, Num::Float
-        rule /\d+\.\d+(e[+-]?\d+)?/i, Num::Float
-        rule /\d+/, Num::Integer
+        rule %r/0x[\da-f]+/i, Num::Hex
+        rule %r/\d+e[+-]?\d+/i, Num::Float
+        rule %r/\d+\.\d+(e[+-]?\d+)?/i, Num::Float
+        rule %r/\d+/, Num::Integer
 
         # Punctuation: [ ] ( ) , ; ` { } :
-        rule /[\[\](),;`{}:]/, Punctuation
+        rule %r/[\[\](),;`{}:]/, Punctuation
       end
 
       # Multiline and nested commenting
       state :multiline_comment do
-        rule /-}/, Comment::Multiline, :pop!
-        rule /{-/, Comment::Multiline, :multiline_comment
-        rule /[^-{}]+/, Comment::Multiline
-        rule /[-{}]/, Comment::Multiline
+        rule %r/-}/, Comment::Multiline, :pop!
+        rule %r/{-/, Comment::Multiline, :multiline_comment
+        rule %r/[^-{}]+/, Comment::Multiline
+        rule %r/[-{}]/, Comment::Multiline
       end
 
       # Double quotes
       state :double_quote do
-        rule /[^\\"]+/, Str::Double
-        rule /\\"/, Str::Escape
-        rule /"/, Str::Double, :pop!
+        rule %r/[^\\"]+/, Str::Double
+        rule %r/\\"/, Str::Escape
+        rule %r/"/, Str::Double, :pop!
       end
 
       # Multiple line string with tripple double quotes, e.g. """ multi """
       state :multiline_string do
-        rule /\s*"""/, Str, :pop!
-        rule /.*/, Str
-        rule /\s*/, Str
+        rule %r/\s*"""/, Str, :pop!
+        rule %r/.*/, Str
+        rule %r/\s*/, Str
       end
 
     end

data/lib/rouge/lexers/epp.rb

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*- #
+
+module Rouge
+  module Lexers
+    class EPP < TemplateLexer
+      title "EPP"
+      desc "Embedded Puppet template files"
+
+      tag 'epp'
+
+      filenames '*.epp'
+
+      def initialize(opts={})
+        super(opts)
+        @parent = lexer_option(:parent) { PlainText.new(opts) }
+        @puppet_lexer = Puppet.new(opts)
+      end
+
+      start do
+        parent.reset!
+        @puppet_lexer.reset!
+      end
+
+      open  = /<%%|<%=|<%#|(<%-|<%)(\s*\|)?/
+      close = /%%>|(\|\s*)?(-%>|%>)/
+
+      state :root do
+        rule %r/<%#/, Comment, :comment
+
+        rule open, Comment::Preproc, :puppet
+
+        rule %r/.+?(?=#{open})|.+/m do
+          delegate parent
+        end
+      end
+
+      state :comment do
+        rule close, Comment, :pop!
+        rule %r/.+?(?=#{close})|.+/m, Comment
+      end
+
+      state :puppet do
+        rule close, Comment::Preproc, :pop!
+
+        rule %r/.+?(?=#{close})|.+/m do
+          delegate @puppet_lexer
+        end
+      end
+    end
+  end
+end

data/lib/rouge/lexers/erb.rb

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*- #
+# frozen_string_literal: true
 
 module Rouge
   module Lexers
@@ -26,24 +27,24 @@ module Rouge
       close = /%%>|-%>|%>/
 
       state :root do
-        rule /<%#/, Comment, :comment
+        rule %r/<%#/, Comment, :comment
 
         rule open, Comment::Preproc, :ruby
 
-        rule /.+?(?=#{open})|.+/m do
+        rule %r/.+?(?=#{open})|.+/m do
           delegate parent
         end
       end
 
       state :comment do
         rule close, Comment, :pop!
-        rule /.+?(?=#{close})|.+/m, Comment
+        rule %r/.+?(?=#{close})|.+/m, Comment
       end
 
       state :ruby do
         rule close, Comment::Preproc, :pop!
 
-        rule /.+?(?=#{close})|.+/m do
+        rule %r/.+?(?=#{close})|.+/m do
           delegate @ruby_lexer
         end
       end

data/lib/rouge/lexers/erlang.rb

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*- #
+# frozen_string_literal: true
 
 module Rouge
   module Lexers

data/lib/rouge/lexers/escape.rb

@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    class Escape < Lexer
+      tag 'escape'
+      aliases 'esc'
+
+      desc 'A generic lexer for including escaped content - see Formatter.enable_escape!'
+
+      option :start, 'the beginning of the escaped section, default "<!"'
+      option :end, 'the end of the escaped section, e.g. "!>"'
+      option :lang, 'the language to lex in unescaped sections'
+
+      attr_reader :start
+      attr_reader :end
+      attr_reader :lang
+
+      def initialize(*)
+        super
+        @start = string_option(:start) { '<!' }
+        @end = string_option(:end) { '!>' }
+        @lang = lexer_option(:lang) { PlainText.new }
+      end
+
+      def to_start_regex
+        @to_start_regex ||= /(.*?)(#{Regexp.escape(@start)})/m
+      end
+
+      def to_end_regex
+        @to_end_regex ||= /(.*?)(#{Regexp.escape(@end)})/m
+      end
+
+      def stream_tokens(str, &b)
+        stream = StringScanner.new(str)
+
+        loop do
+          if stream.scan(to_start_regex)
+            puts "pre-escape: #{stream[1].inspect}" if @debug
+            @lang.continue_lex(stream[1], &b)
+          else
+            # no more start delimiters, scan til the end
+            @lang.continue_lex(stream.rest, &b)
+            return
+          end
+
+          if stream.scan(to_end_regex)
+            yield Token::Tokens::Escape, stream[1]
+          else
+            yield Token::Tokens::Escape, stream.rest
+            return
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rouge/lexers/factor.rb

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*- #
+# frozen_string_literal: true
 
 module Rouge
   module Lexers
@@ -166,35 +167,35 @@ module Rouge
       end
 
       state :root do
-        rule /\s+/m, Text
+        rule %r/\s+/m, Text
 
-        rule /(:|::|MACRO:|MEMO:|GENERIC:|HELP:)(\s+)(\S+)/m do
+        rule %r/(:|::|MACRO:|MEMO:|GENERIC:|HELP:)(\s+)(\S+)/m do
           groups Keyword, Text, Name::Function
         end
 
-        rule /(M:|HOOK:|GENERIC#)(\s+)(\S+)(\s+)(\S+)/m do
+        rule %r/(M:|HOOK:|GENERIC#)(\s+)(\S+)(\s+)(\S+)/m do
           groups Keyword, Text, Name::Class, Text, Name::Function
         end
 
-        rule /\((?=\s)/, Name::Function, :stack_effect
-        rule /;(?=\s)/, Keyword
+        rule %r/\((?=\s)/, Name::Function, :stack_effect
+        rule %r/;(?=\s)/, Keyword
 
-        rule /(USING:)((?:\s|\\\s)+)/m do
+        rule %r/(USING:)((?:\s|\\\s)+)/m do
           groups Keyword::Namespace, Text
           push :import
         end
 
-        rule /(IN:|USE:|UNUSE:|QUALIFIED:|QUALIFIED-WITH:)(\s+)(\S+)/m do
+        rule %r/(IN:|USE:|UNUSE:|QUALIFIED:|QUALIFIED-WITH:)(\s+)(\S+)/m do
           groups Keyword::Namespace, Text, Name::Namespace
         end
 
-        rule /(FROM:|EXCLUDE:)(\s+)(\S+)(\s+)(=>)/m do
+        rule %r/(FROM:|EXCLUDE:)(\s+)(\S+)(\s+)(=>)/m do
           groups Keyword::Namespace, Text, Name::Namespace, Text, Punctuation
         end
 
-        rule /(?:ALIAS|DEFER|FORGET|POSTPONE):/, Keyword::Namespace
+        rule %r/(?:ALIAS|DEFER|FORGET|POSTPONE):/, Keyword::Namespace
 
-        rule /(TUPLE:)(\s+)(\S+)(\s+)(<)(\s+)(\S+)/m do
+        rule %r/(TUPLE:)(\s+)(\S+)(\s+)(<)(\s+)(\S+)/m do
           groups(
             Keyword, Text,
             Name::Class, Text,
@@ -204,16 +205,16 @@ module Rouge
           push :slots
         end
 
-        rule /(TUPLE:)(\s+)(\S+)/m do
+        rule %r/(TUPLE:)(\s+)(\S+)/m do
           groups Keyword, Text, Name::Class
           push :slots
         end
 
-        rule /(UNION:|INTERSECTION:)(\s+)(\S+)/m do
+        rule %r/(UNION:|INTERSECTION:)(\s+)(\S+)/m do
           groups Keyword, Text, Name::Class
         end
 
-        rule /(PREDICATE:)(\s+)(\S+)(\s+)(<)(\s+)(\S+)/m do
+        rule %r/(PREDICATE:)(\s+)(\S+)(\s+)(<)(\s+)(\S+)/m do
           groups(
             Keyword, Text,
             Name::Class, Text,
@@ -222,7 +223,7 @@ module Rouge
           )
         end
 
-        rule /(C:)(\s+)(\S+)(\s+)(\S+)/m do
+        rule %r/(C:)(\s+)(\S+)(\s+)(\S+)/m do
           groups(
             Keyword, Text,
             Name::Function, Text,
@@ -235,38 +236,38 @@ module Rouge
            |ALIEN|TYPEDEF|FUNCTION|STRUCT):
         )x, Keyword
 
-        rule /(?:<PRIVATE|PRIVATE>)/, Keyword::Namespace
+        rule %r/(?:<PRIVATE|PRIVATE>)/, Keyword::Namespace
 
-        rule /(MAIN:)(\s+)(\S+)/ do
+        rule %r/(MAIN:)(\s+)(\S+)/ do
           groups Keyword::Namespace, Text, Name::Function
         end
 
         # strings
-        rule /"""\s+.*?\s+"""/, Str
-        rule /"(\\.|[^\\])*?"/, Str
-        rule /(CHAR:)(\s+)(\\[\\abfnrstv]*|\S)(?=\s)/, Str::Char
+        rule %r/"""\s+.*?\s+"""/, Str
+        rule %r/"(\\.|[^\\])*?"/, Str
+        rule %r/(CHAR:)(\s+)(\\[\\abfnrstv]*|\S)(?=\s)/, Str::Char
 
         # comments
-        rule /!\s+.*$/, Comment
-        rule /#!\s+.*$/, Comment
+        rule %r/!\s+.*$/, Comment
+        rule %r/#!\s+.*$/, Comment
 
         # booleans
-        rule /[tf](?=\s)/, Name::Constant
+        rule %r/[tf](?=\s)/, Name::Constant
 
         # numbers
-        rule /-?\d+\.\d+(?=\s)/, Num::Float
-        rule /-?\d+(?=\s)/, Num::Integer
+        rule %r/-?\d+\.\d+(?=\s)/, Num::Float
+        rule %r/-?\d+(?=\s)/, Num::Integer
 
-        rule /HEX:\s+[a-fA-F\d]+(?=\s)/m, Num::Hex
-        rule /BIN:\s+[01]+(?=\s)/, Num::Bin
-        rule /OCT:\s+[0-7]+(?=\s)/, Num::Oct
+        rule %r/HEX:\s+[a-fA-F\d]+(?=\s)/m, Num::Hex
+        rule %r/BIN:\s+[01]+(?=\s)/, Num::Bin
+        rule %r/OCT:\s+[0-7]+(?=\s)/, Num::Oct
 
         rule %r([-+/*=<>^](?=\s)), Operator
 
-        rule /(?:deprecated|final|foldable|flushable|inline|recursive)(?=\s)/,
+        rule %r/(?:deprecated|final|foldable|flushable|inline|recursive)(?=\s)/,
           Keyword
 
-        rule /\S+/ do |m|
+        rule %r/\S+/ do |m|
           name = m[0]
 
           if self.class.builtins.values.any? { |b| b.include? name }
@@ -278,24 +279,24 @@ module Rouge
       end
 
       state :stack_effect do
-        rule /\s+/, Text
-        rule /\(/, Name::Function, :stack_effect
-        rule /\)/, Name::Function, :pop!
+        rule %r/\s+/, Text
+        rule %r/\(/, Name::Function, :stack_effect
+        rule %r/\)/, Name::Function, :pop!
 
-        rule /--/, Name::Function
-        rule /\S+/, Name::Variable
+        rule %r/--/, Name::Function
+        rule %r/\S+/, Name::Variable
       end
 
       state :slots do
-        rule /\s+/, Text
-        rule /;(?=\s)/, Keyword, :pop!
-        rule /\S+/, Name::Variable
+        rule %r/\s+/, Text
+        rule %r/;(?=\s)/, Keyword, :pop!
+        rule %r/\S+/, Name::Variable
       end
 
       state :import do
-        rule /;(?=\s)/, Keyword, :pop!
-        rule /\s+/, Text
-        rule /\S+/, Name::Namespace
+        rule %r/;(?=\s)/, Keyword, :pop!
+        rule %r/\s+/, Text
+        rule %r/\S+/, Name::Namespace
       end
     end
   end