rouge 4.0.0 → 4.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9229545bdc0804f0a7f0fea6a4fd1b94dbe8d022d0d29c1ffa254c09f42003d1
-  data.tar.gz: cf12e8de0f2d03d37ed6f1380a5deb04ad385d8628b4f2ed7c1921b16161636f
+  metadata.gz: 555fb3207f31723860f17a269418bbaea9dff23c0ae97dc5f27fabd797fc4ad5
+  data.tar.gz: ad0c9a8de4f66de72f487f8310abda1e763c92816ef2de031a7aea89bcd155c0
 SHA512:
-  metadata.gz: 31f64a3f386d01829f9a2d2e9711ad40c3c48fbf122dc66b0e9519d4c02676cf530879c388e280efd996d6a57ec3497f5faa54423d081af02450348037c58712
-  data.tar.gz: fad76c2d5d770fb172e09a2614e37ff78d362b81dfb73d50ce5c25de04f07c93aabae357e4a1a5b6b47be3792edf146e1299c1ec78204d55e998dd54c169f307
+  metadata.gz: fcdb09306c5640839523e4ade20a7a272d835f0cd7eec7d2a3b43ae4598d5f0ef50ce2c64e24de31d98e10501f5a8ea53b0e064b51a0e3166756a0669e76b3e9
+  data.tar.gz: 8c2deba5fc1d897f655642ed01f1817743d249fe1a8dabcc9de873b5203378b63315c5f5fd18c25a479d97aed5e726c9efffb43b1f0c83fbcc6aaf75ce1cf23d

data/Gemfile

@@ -8,7 +8,7 @@ gem 'rake'
 
 gem 'minitest', '>= 5.0'
 gem 'minitest-power_assert'
-gem 'power_assert', '~> 1.2'
+gem 'power_assert', '~> 2.0'
 
 gem 'rubocop', '~> 1.0', '<= 1.11'
 gem 'rubocop-performance'

data/lib/rouge/demos/cisco_ios

@@ -0,0 +1,19 @@
+interface FastEthernet0.20
+    encapsulation dot1Q 20
+    no ip route-cache
+    bridge-group 1
+    no bridge-group 1 source-learning
+    bridge-group 1 spanning-disabled
+
+! Supports shortened versions of config words, too
+inter gi0.10
+encap dot1q 10 native
+inter gi0
+
+banner login # Authenticate yourself! #
+
+! Supports #, $ and % to delimit banners, and multiline
+banner motd $
+Attention!
+We will be having scheduled system maintenance on this device.
+$

data/lib/rouge/demos/coq

@@ -10,4 +10,7 @@ Section with_T.
     end.
 End with_T.
 
-Definition a_string := "hello \" world".
+Definition a_string := "hello
+world".
+Definition escape_string := "0123".
+Definition zero_string := "0".

data/lib/rouge/formatters/html.rb

@@ -5,6 +5,14 @@ module Rouge
   module Formatters
     # Transforms a token stream into HTML output.
     class HTML < Formatter
+      TABLE_FOR_ESCAPE_HTML = {
+        '&' => '&amp;',
+        '<' => '&lt;',
+        '>' => '&gt;',
+      }.freeze
+
+      ESCAPE_REGEX = /[&<>]/.freeze
+
       tag 'html'
 
       # @yield the html output.
@@ -22,20 +30,14 @@ module Rouge
         if tok == Token::Tokens::Text
           safe_val
         else
-          shortname = tok.shortname \
-            or raise "unknown token: #{tok.inspect} for #{safe_val.inspect}"
+          shortname = tok.shortname or raise "unknown token: #{tok.inspect} for #{safe_val.inspect}"
 
           "<span class=\"#{shortname}\">#{safe_val}</span>"
         end
       end
 
-      TABLE_FOR_ESCAPE_HTML = {
-        '&' => '&amp;',
-        '<' => '&lt;',
-        '>' => '&gt;',
-      }
+      private
 
-    private
       # A performance-oriented helper method to escape `&`, `<` and `>` for the rendered
       # HTML from this formatter.
       #
@@ -46,10 +48,9 @@ module Rouge
       # Returns either the given `value` argument string as is or a new string with the
       # special characters replaced with their escaped counterparts.
       def escape_special_html_chars(value)
-        escape_regex = /[&<>]/
-        return value unless value =~ escape_regex
+        return value unless value =~ ESCAPE_REGEX
 
-        value.gsub(escape_regex, TABLE_FOR_ESCAPE_HTML)
+        value.gsub(ESCAPE_REGEX, TABLE_FOR_ESCAPE_HTML)
       end
     end
   end

data/lib/rouge/guessers/disambiguation.rb

@@ -133,7 +133,8 @@ module Rouge
       end
 
       disambiguate '*.pp' do
-        next Pascal if matches?(/\b(function|begin|var)\b/)
+        next Puppet if matches?(/(::)?([a-z]\w*::)/)
+        next Pascal if matches?(/^(function|begin|var)\b/)
         next Pascal if matches?(/\b(end(;|\.))/)
 
         Puppet

data/lib/rouge/lexers/cisco_ios.rb

@@ -0,0 +1,83 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+# Based on/regexes mostly from Brandon Bennett's pygments-routerlexers:
+# https://github.com/nemith/pygments-routerlexers
+
+module Rouge
+  module Lexers
+    class CiscoIos < RegexLexer
+      title 'Cisco IOS'
+      desc 'Cisco IOS configuration lexer'
+      tag 'cisco_ios'
+      filenames '*.cfg'
+      mimetypes 'text/x-cisco-conf'
+
+      state :root do
+        rule %r/^!.*/, Comment::Single
+
+        rule %r/^(version\s+)(.*)$/ do
+          groups Keyword, Num::Float
+        end
+
+        rule %r/(desc*r*i*p*t*i*o*n*)(.*?)$/ do
+          groups Keyword, Comment::Single
+        end
+
+        rule %r/^(inte*r*f*a*c*e*|controller|router \S+|voice translation-\S+|voice-port|line)(.*)$/ do
+          groups Keyword::Type, Name::Function
+        end
+
+        rule %r/(password|secret)(\s+[57]\s+)(\S+)/ do
+          groups Keyword, Num, String::Double
+        end
+
+        rule %r/(permit|deny)/, Operator::Word
+
+        rule %r/^(banner\s+)(motd\s+|login\s+)([#$%])/ do
+          groups Keyword, Name::Function, Str::Delimiter
+          push :cisco_ios_text
+        end
+
+        rule %r/^(dial-peer\s+\S+\s+)(\S+)(.*?)$/ do
+          groups Keyword, Name::Attribute, Keyword
+        end
+
+        rule %r/^(vlan\s+)(\d+)$/ do
+          groups Keyword, Name::Attribute
+        end
+
+        # IPv4 Address/Prefix
+        rule %r/(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})(\/\d{1,2})?/, Num
+
+        # NSAP
+        rule %r/49\.\d{4}\.\d{4}\.\d{4}\.\d{4}\.\d{2}/, Num
+
+        # MAC Address
+        rule %r/[a-f0-9]{4}\.[a-f0-9]{4}\.[a-f0-9]{4}/, Num::Hex
+
+        rule %r/^(\s*no\s+)(\S+)/ do
+          groups Keyword::Constant, Keyword
+        end
+
+        rule %r/^[^\n\r]\s*\S+/, Keyword
+
+        # Obfuscated Passwords
+        rule  %r/\*+/, Name::Entity
+
+        rule %r/(?<= )\d+(?= )/, Num
+
+        # Newline catcher, avoid errors on empty lines
+        rule %r/\n+/m, Text
+
+        # This one goes last, a text catch-all
+        rule %r/./, Text
+      end
+
+      state :cisco_ios_text do
+        rule %r/[^#$%]/, Text
+        rule %r/[#$%]/, Str::Delimiter, :pop!
+      end
+    end
+  end
+end

data/lib/rouge/lexers/coq.rb

@@ -25,7 +25,7 @@ module Rouge
           Variables Class Instance Global Local Include
           Printing Notation Infix Arguments Hint Rewrite Immediate
           Qed Defined Opaque Transparent Existing
-          Compute Eval Print SearchAbout Search About Check
+          Compute Eval Print SearchAbout Search About Check Admitted
         )
       end
 
@@ -56,16 +56,6 @@ module Rouge
         )
       end
 
-      def self.keyopts
-        @keyopts ||= Set.new %w(
-          := => -> /\\ \\/ _ ; :> : ⇒ → ↔ ⇔ ≔ ≡ ∀ ∃ ∧ ∨ ¬ ⊤ ⊥ ⊢ ⊨ ∈
-        )
-      end
-
-      def self.end_sentence
-        @end_sentence ||= Punctuation::Indicator
-      end
-
       def self.classify(x)
         if self.coq.include? x
           return Keyword
@@ -82,58 +72,133 @@ module Rouge
         end
       end
 
-      operator = %r([\[\];,{}_()!$%&*+./:<=>?@^|~#-]+)
-      id = /(?:[a-z][\w']*)|(?:[_a-z][\w']+)/i
-      dot_id = /\.((?:[a-z][\w']*)|(?:[_a-z][\w']+))/i
+      # https://github.com/coq/coq/blob/110921a449fcb830ec2a1cd07e3acc32319feae6/clib/unicode.ml#L67
+      # https://coq.inria.fr/refman/language/core/basic.html#grammar-token-ident
+      id_first = /\p{L}/
+      id_first_underscore = /(?:\p{L}|_)/
+      id_subsequent = /(?:\p{L}|\p{N}|_|')/ # a few missing? some mathematical ' primes and subscripts
+      id = /(?:#{id_first}#{id_subsequent}*)|(?:#{id_first_underscore}#{id_subsequent}+)/i
+      dot_id = /\.(#{id})/i
       dot_space = /\.(\s+)/
-      module_type = /Module(\s+)Type(\s+)/
-      set_options = /(Set|Unset)(\s+)(Universe|Printing|Implicit|Strict)(\s+)(Polymorphism|All|Notations|Arguments|Universes|Implicit)(\s*)\./m
 
       state :root do
-        rule %r/[(][*](?![)])/, Comment, :comment
+        mixin :begin_proof
+        mixin :sentence
+      end
+
+      state :sentence do
+        mixin :comment_whitespace
+        mixin :module_setopts
+        # After parsing the id, end up in sentence_postid
+        rule id do |m|
+          @name = m[0]
+          @id_dotted = false
+          push :sentence_postid
+          push :continue_id
+        end
+      end
+
+      state :begin_proof do
+        rule %r/(Proof)(\s*)(\.)(\s+)/i do
+          groups Keyword, Text::Whitespace, Punctuation::Indicator, Text::Whitespace
+          push :proof_mode
+        end
+      end
+
+      state :proof_mode do
+        mixin :comment_whitespace
+        mixin :module_setopts
+        mixin :begin_proof
+
+        rule %r/(Qed|Defined|Save|Admitted)(\s*)(\.)(\s+)/i do
+          groups Keyword, Text::Whitespace, Punctuation::Indicator, Text::Whitespace
+          pop!
+        end
+        # the whole point of parsing Proof/Qed, normally some of these will be operators
+        rule %r/(?:\-+|\++|\*+)/, Punctuation
+        rule %r/[{}]/, Punctuation
+        # toplevel_selector
+        rule %r/(!|all|par)(:)/ do
+          groups Keyword::Pseudo, Punctuation
+        end
+        # numbered goals 1: {} 1,2: {}
+        rule %r/\d+/, Num::Integer, :numeric_labels
+        # [named_goal]: { ... }
+        rule %r/(\[)(\s*)(#{id})(\s*)(\])(\s*)(:)/ do
+          groups Punctuation, Text::Whitespace, Name::Constant, Text::Whitespace, Punctuation, Text::Whitespace, Punctuation
+        end
+        # After parsing the id, end up in sentence_postid
+        rule id do |m|
+          @name = m[0]
+          @id_dotted = false
+          push :sentence_postid
+          push :continue_id
+        end
+      end
+
+      state :numeric_labels do
+        mixin :whitespace
+        rule %r/(,)(\s*)(\d+)/ do
+          groups Punctuation, Text::Whitespace, Num::Integer
+        end
+
+        rule %r(:), Punctuation, :pop!
+      end
+
+      state :whitespace do
         rule %r/\s+/m, Text::Whitespace
-        rule module_type do |m|
-          token Keyword , 'Module'
-          token Text::Whitespace , m[1]
-          token Keyword , 'Type'
-          token Text::Whitespace , m[2]
+      end
+
+      state :comment_whitespace do
+        rule %r/[(][*](?![)])/, Comment, :comment
+        mixin :whitespace
+      end
+
+      state :module_setopts do
+        rule %r/(Module)(\s+)(Type)(\s+)/ do
+          groups Keyword, Text::Whitespace, Keyword, Text::Whitespace
         end
-        rule set_options do |m|
-          token Keyword , m[1]
-          i = 2
-          while m[i] != ''
-            token Text::Whitespace , m[i]
-            token Keyword , m[i+1]
-            i += 2
-          end
-          token self.class.end_sentence , '.'
+
+        rule %r(
+          (Set|Unset)(\s+)
+          (Universe|Printing|Implicit|Strict)(\s+)
+          (Polymorphism|All|Notations|Arguments|Universes|Implicit)?(\s*)(\.)
+        )x do
+          groups Keyword, Text::Whitespace, Keyword, Text::Whitespace, Keyword, Text::Whitespace, Punctuation::Indicator
         end
+      end
+
+      state :sentence_postid do
+        mixin :comment_whitespace
+        mixin :module_setopts
+
+        # up here to beat the id rule for lambda
+        rule %r(:=|=>|;|:>|:|::|_), Punctuation
+        rule %r(->|/\\|\\/|;|:>|[⇒→↔⇔≔≡∀∃∧∨¬⊤⊥⊢⊨∈λ]), Operator
+
         rule id do |m|
           @name = m[0]
-          @continue = false
+          @id_dotted = false
           push :continue_id
         end
-        rule %r(/\\), Operator
-        rule %r/\\\//, Operator
+
+        # must be followed by whitespace, so that we don't match notations like sym.(a + b)
+        rule %r/\.(?=\s)/, Punctuation::Indicator, :pop! # :sentence_postid
 
         rule %r/-?\d[\d_]*(.[\d_]*)?(e[+-]?\d[\d_]*)/i, Num::Float
-        rule %r/\d[\d_]*/, Num::Integer
+        rule %r/-?\d[\d_]*/, Num::Integer
 
         rule %r/'(?:(\\[\\"'ntbr ])|(\\[0-9]{3})|(\\x\h{2}))'/, Str::Char
         rule %r/'/, Keyword
         rule %r/"/, Str::Double, :string
         rule %r/[~?]#{id}/, Name::Variable
 
-        rule %r/./ do |m|
-          match = m[0]
-          if self.class.keyopts.include? match
-            token Punctuation
-          elsif match =~ operator
-            token Operator
-          else
-            token Error
-          end
-        end
+        rule %r(`{|[{}\[\]()?|;,.]), Punctuation
+        rule %r([!@^|~#.%/]+), Operator
+        # any other combo of S (symbol), P (punctuation) and some extras just to be sure
+        rule %r((?:\p{S}|\p{Pc}|[./\:\<=>\-+*])+), Operator
+
+        rule %r/./, Error
       end
 
       state :comment do
@@ -144,48 +209,47 @@ module Rouge
       end
 
       state :string do
-        rule %r/(?:\\")+|[^"]/, Str::Double
-        mixin :escape_sequence
-        rule %r/\\\n/, Str::Double
+        rule %r/[^"]+/, Str::Double
+        rule %r/""/, Str::Double
         rule %r/"/, Str::Double, :pop!
       end
 
-      state :escape_sequence do
-        rule %r/\\[\\"'ntbr]/, Str::Escape
-      end
-
       state :continue_id do
         # the stream starts with an id (stored in @name) and continues here
         rule dot_id do |m|
-          token Name::Namespace , @name
-          token Punctuation , '.'
-          @continue = true
+          token Name::Namespace, @name
+          token Punctuation, '.'
+          @id_dotted = true
           @name = m[1]
         end
+
         rule dot_space do |m|
-          if @continue
-            token Name::Constant , @name
+          if @id_dotted
+            token Name::Constant, @name
           else
-            token self.class.classify(@name) , @name
+            token self.class.classify(@name), @name
           end
-          token self.class.end_sentence , '.'
-          token Text::Whitespace , m[1]
+
+          token Punctuation::Indicator, '.'
+          token Text::Whitespace, m[1]
           @name = false
-          @continue = false
-          pop!
+          @id_dotted = false
+          pop! # :continue_id
+          pop! # :sentence_postid
         end
+
         rule %r// do
-          if @continue
-            token Name::Constant , @name
+          if @id_dotted
+            token Name::Constant, @name
           else
-            token self.class.classify(@name) , @name
+            token self.class.classify(@name), @name
           end
           @name = false
-          @continue = false
-          pop!
+          @id_dotted = false
+          # we finished parsing an id, drop back into the sentence_postid that was pushed first.
+          pop! # :continue_id
         end
       end
-
     end
   end
 end

data/lib/rouge/lexers/cpp.rb

@@ -71,7 +71,11 @@ module Rouge
         rule %r/\bnullptr\b/, Name::Builtin
         rule %r/(?:u8|u|U|L)?R"([a-zA-Z0-9_{}\[\]#<>%:;.?*\+\-\/\^&|~!=,"']{,16})\(.*?\)\1"/m, Str
         rule %r/(::|<=>)/, Operator
-        rule %r/[{}]/, Punctuation
+        rule %r/[{]/, Punctuation
+        rule %r/}/ do
+          token Punctuation
+          pop! if in_state?(:function) # pop :function
+        end
       end
 
       state :classname do

data/lib/rouge/lexers/csharp.rb

@@ -27,19 +27,19 @@ module Rouge
         static switch this throw true try typeof unchecked unsafe
         virtual void volatile while
         add alias async await get global partial remove set value where
-        yield nameof
-        ascending by descending equals from group in into join let on
-        orderby select
+        yield nameof notnull
+        ascending by descending equals from group in init into join let
+        on orderby select unmanaged when and not or with
       )
 
       keywords_type = %w(
-        bool byte char decimal double dynamic float int long object
-        sbyte short string uint ulong ushort var
+        bool byte char decimal double dynamic float int long nint nuint
+        object sbyte short string uint ulong ushort var
       )
 
       cpp_keywords = %w(
         if endif else elif define undef line error warning region
-        endregion pragma
+        endregion pragma nullable
       )
 
       state :whitespace do
@@ -81,14 +81,15 @@ module Rouge
         rule %r/@"(""|[^"])*"/m, Str
         rule %r/"(\\.|.)*?["\n]/, Str
         rule %r/'(\\.|.)'/, Str::Char
-        rule %r/0x[0-9a-f]+[lu]?/i, Num
+        rule %r/0b[_01]+[lu]?/i, Num
+        rule %r/0x[_0-9a-f]+[lu]?/i, Num
         rule %r(
-          [0-9]
-          ([.][0-9]*)? # decimal
-          (e[+-][0-9]+)? # exponent
-          [fldu]? # type
+          [0-9](?:[_0-9]*[0-9])?
+          ([.][0-9](?:[_0-9]*[0-9])?)? # decimal
+          (e[+-]?[0-9](?:[_0-9]*[0-9])?)? # exponent
+          [fldum]? # type
         )ix, Num
-        rule %r/\b(?:class|struct|interface)\b/, Keyword, :class
+        rule %r/\b(?:class|record|struct|interface)\b/, Keyword, :class
         rule %r/\b(?:namespace|using)\b/, Keyword, :namespace
         rule %r/^#[ \t]*(#{cpp_keywords.join('|')})\b.*?\n/,
           Comment::Preproc

data/lib/rouge/lexers/ghc_cmm.rb

@@ -22,7 +22,7 @@ module Rouge
       ws = %r(\s|//.*?\n|/[*](?:[^*]|(?:[*][^/]))*[*]+/)mx
 
       # Make sure that this is not a preprocessor macro, e.g. `#if` or `#define`.
-      id = %r((?!#[a-zA-Z])[\w#\$%_']+)
+      id = %r((?!\#[a-zA-Z])[\w#\$%_']+)
 
       complex_id = %r(
         (?:[\w#$%_']|\(\)|\(,\)|\[\]|[0-9])*