rouge 3.20.0 → 3.25.0

data/lib/rouge/lexers/wollok.rb

@@ -18,7 +18,6 @@ module Rouge
 
       state :whitespaces_and_comments do
         rule %r/\s+/m, Text::Whitespace
-        rule %r/$+/m, Text::Whitespace
         rule %r(//.*$), Comment::Single
         rule %r(/\*(.|\s)*?\*/)m, Comment::Multiline
       end

data/lib/rouge/lexers/xml.rb

@@ -12,6 +12,8 @@ module Rouge
       mimetypes 'text/xml', 'application/xml', 'image/svg+xml',
                 'application/rss+xml', 'application/atom+xml'
 
+      # Documentation: https://www.w3.org/TR/xml11/#charsets and https://www.w3.org/TR/xml11/#sec-suggested-names
+
       def self.detect?(text)
         return false if text.doctype?(/html/)
         return true if text =~ /\A<\?xml\b/
@@ -27,10 +29,10 @@ module Rouge
         rule %r/<![^>]*>/, Comment::Preproc
 
         # open tags
-        rule %r(<\s*[\w:.-]+)m, Name::Tag, :tag
+        rule %r(<\s*[\p{L}:_][\p{Word}\p{Cf}:.·-]*)m, Name::Tag, :tag
 
         # self-closing tags
-        rule %r(<\s*/\s*[\w:.-]+\s*>)m, Name::Tag
+        rule %r(<\s*/\s*[\p{L}:_][\p{Word}\p{Cf}:.·-]*\s*>)m, Name::Tag
       end
 
       state :comment do
@@ -41,7 +43,7 @@ module Rouge
 
       state :tag do
         rule %r/\s+/m, Text
-        rule %r/[\w.:-]+\s*=/m, Name::Attribute, :attr
+        rule %r/[\p{L}:_][\p{Word}\p{Cf}:.·-]*\s*=/m, Name::Attribute, :attr
         rule %r(/?\s*>), Name::Tag, :pop!
       end
 

data/lib/rouge/lexers/yaml.rb

@@ -11,6 +11,8 @@ module Rouge
       aliases 'yml'
       filenames '*.yaml', '*.yml'
 
+      # Documentation: https://yaml.org/spec/1.2/spec.html
+
       def self.detect?(text)
         # look for the %YAML directive
         return true if text =~ /\A\s*%YAML/m
@@ -165,15 +167,15 @@ module Rouge
         )x, Keyword::Type
 
         # an anchor
-        rule %r/&[\w-]+/, Name::Label
+        rule %r/&[\p{L}\p{Nl}\p{Nd}_-]+/, Name::Label
 
         # an alias
-        rule %r/\*[\w-]+/, Name::Variable
+        rule %r/\*[\p{L}\p{Nl}\p{Nd}_-]+/, Name::Variable
       end
 
       state :block_nodes do
         # implicit key
-        rule %r/((?:\w[\w -]*)?)(:)(?=\s|$)/ do |m|
+        rule %r/((?:[\p{L}\p{Nl}\p{Nd}_][\p{L}\p{Nl}\p{Nd}\p{Blank}_-]*)?)(:)(?=\s|$)/ do |m|
           groups Name::Attribute, Punctuation::Indicator
           set_indent m[0], :implicit => true
         end

data/lib/rouge/regex_lexer.rb

@@ -6,6 +6,37 @@ module Rouge
   # A stateful lexer that uses sets of regular expressions to
   # tokenize a string.  Most lexers are instances of RegexLexer.
   class RegexLexer < Lexer
+    class InvalidRegex < StandardError
+      def initialize(re)
+        @re = re
+      end
+
+      def to_s
+        "regex #{@re.inspect} matches empty string, but has no predicate!"
+      end
+    end
+
+    class ClosedState < StandardError
+      attr_reader :state
+      def initialize(state)
+        @state = state
+      end
+
+      def rule
+        @state.rules.last
+      end
+
+      def to_s
+        rule = @state.rules.last
+        msg = "State :#{state.name} cannot continue after #{rule.inspect}, which will always match."
+        if rule.re.source.include?('*')
+          msg += " Consider replacing * with +."
+        end
+
+        msg
+      end
+    end
+
     # A rule is a tuple of a regular expression to test, and a callback
     # to perform if the test succeeds.
     #
@@ -42,12 +73,13 @@ module Rouge
     end
 
     class StateDSL
-      attr_reader :rules
+      attr_reader :rules, :name
       def initialize(name, &defn)
         @name = name
         @defn = defn
         @rules = []
         @loaded = false
+        @closed = false
       end
 
       def to_state(lexer_class)
@@ -95,10 +127,14 @@ module Rouge
       #   {RegexLexer#token}, and {RegexLexer#delegate}.  The first
       #   argument can be used to access the match groups.
       def rule(re, tok=nil, next_state=nil, &callback)
+        raise ClosedState.new(self) if @closed
+
         if tok.nil? && callback.nil?
           raise "please pass `rule` a token to yield or a callback"
         end
 
+        matches_empty = re =~ ''
+
         callback ||= case next_state
         when :pop!
           proc do |stream|
@@ -123,6 +159,9 @@ module Rouge
             @stack.push(state)
           end
         when nil
+          # cannot use an empty-matching regexp with no predicate
+          raise InvalidRegex.new(re) if matches_empty
+
           proc do |stream|
             puts "    yielding: #{tok.qualname}, #{stream[0].inspect}" if @debug
             @output_stream.call(tok, stream[0])
@@ -132,6 +171,22 @@ module Rouge
         end
 
         rules << Rule.new(re, callback)
+
+        close! if matches_empty && !context_sensitive?(re)
+      end
+
+      def context_sensitive?(re)
+        source = re.source
+        return true if source =~ /[(][?]<?[!=]/
+
+        # anchors count as lookahead/behind
+        return true if source =~ /[$^]/
+
+        false
+      end
+
+      def close!
+        @closed = true
       end
 
       # Mix in the rules from another state into this state.  The rules

data/lib/rouge/version.rb

@@ -3,6 +3,6 @@
 
 module Rouge
   def self.version
-    "3.20.0"
+    "3.25.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rouge
 version: !ruby/object:Gem::Version
-  version: 3.20.0
+  version: 3.25.0
 platform: ruby
 authors:
 - Jeanine Adkisson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-09 00:00:00.000000000 Z
+date: 2020-11-10 00:00:00.000000000 Z
 dependencies: []
 description: Rouge aims to a be a simple, easy-to-extend drop-in replacement for pygments.
 email:
@@ -39,6 +39,7 @@ files:
 - lib/rouge/demos/biml
 - lib/rouge/demos/bpf
 - lib/rouge/demos/brainfuck
+- lib/rouge/demos/brightscript
 - lib/rouge/demos/bsl
 - lib/rouge/demos/c
 - lib/rouge/demos/ceylon
@@ -72,6 +73,7 @@ files:
 - lib/rouge/demos/eiffel
 - lib/rouge/demos/elixir
 - lib/rouge/demos/elm
+- lib/rouge/demos/email
 - lib/rouge/demos/epp
 - lib/rouge/demos/erb
 - lib/rouge/demos/erlang
@@ -108,6 +110,8 @@ files:
 - lib/rouge/demos/irb
 - lib/rouge/demos/irb_output
 - lib/rouge/demos/isbl
+- lib/rouge/demos/j
+- lib/rouge/demos/janet
 - lib/rouge/demos/java
 - lib/rouge/demos/javascript
 - lib/rouge/demos/jinja
@@ -156,6 +160,7 @@ files:
 - lib/rouge/demos/plaintext
 - lib/rouge/demos/plist
 - lib/rouge/demos/pony
+- lib/rouge/demos/postscript
 - lib/rouge/demos/powershell
 - lib/rouge/demos/praat
 - lib/rouge/demos/prolog
@@ -190,8 +195,10 @@ files:
 - lib/rouge/demos/sparql
 - lib/rouge/demos/sqf
 - lib/rouge/demos/sql
+- lib/rouge/demos/ssh
 - lib/rouge/demos/supercollider
 - lib/rouge/demos/swift
+- lib/rouge/demos/systemd
 - lib/rouge/demos/tap
 - lib/rouge/demos/tcl
 - lib/rouge/demos/terraform
@@ -223,6 +230,7 @@ files:
 - lib/rouge/formatters/html.rb
 - lib/rouge/formatters/html_inline.rb
 - lib/rouge/formatters/html_legacy.rb
+- lib/rouge/formatters/html_line_highlighter.rb
 - lib/rouge/formatters/html_line_table.rb
 - lib/rouge/formatters/html_linewise.rb
 - lib/rouge/formatters/html_pygments.rb
@@ -257,6 +265,7 @@ files:
 - lib/rouge/lexers/biml.rb
 - lib/rouge/lexers/bpf.rb
 - lib/rouge/lexers/brainfuck.rb
+- lib/rouge/lexers/brightscript.rb
 - lib/rouge/lexers/bsl.rb
 - lib/rouge/lexers/c.rb
 - lib/rouge/lexers/ceylon.rb
@@ -290,6 +299,7 @@ files:
 - lib/rouge/lexers/eiffel.rb
 - lib/rouge/lexers/elixir.rb
 - lib/rouge/lexers/elm.rb
+- lib/rouge/lexers/email.rb
 - lib/rouge/lexers/epp.rb
 - lib/rouge/lexers/erb.rb
 - lib/rouge/lexers/erlang.rb
@@ -327,6 +337,8 @@ files:
 - lib/rouge/lexers/irb.rb
 - lib/rouge/lexers/isbl.rb
 - lib/rouge/lexers/isbl/builtins.rb
+- lib/rouge/lexers/j.rb
+- lib/rouge/lexers/janet.rb
 - lib/rouge/lexers/java.rb
 - lib/rouge/lexers/javascript.rb
 - lib/rouge/lexers/jinja.rb
@@ -384,6 +396,7 @@ files:
 - lib/rouge/lexers/plain_text.rb
 - lib/rouge/lexers/plist.rb
 - lib/rouge/lexers/pony.rb
+- lib/rouge/lexers/postscript.rb
 - lib/rouge/lexers/powershell.rb
 - lib/rouge/lexers/praat.rb
 - lib/rouge/lexers/prolog.rb
@@ -420,8 +433,10 @@ files:
 - lib/rouge/lexers/sqf.rb
 - lib/rouge/lexers/sqf/keywords.rb
 - lib/rouge/lexers/sql.rb
+- lib/rouge/lexers/ssh.rb
 - lib/rouge/lexers/supercollider.rb
 - lib/rouge/lexers/swift.rb
+- lib/rouge/lexers/systemd.rb
 - lib/rouge/lexers/tap.rb
 - lib/rouge/lexers/tcl.rb
 - lib/rouge/lexers/terraform.rb