rouge 4.7.0 → 5.1.0

data/lib/rouge/lexers/viml.rb

@@ -13,9 +13,8 @@ module Rouge
 
       mimetypes 'text/x-vim'
 
-      def self.keywords
-        Kernel::load File.join(Lexers::BASE_DIR, 'viml/keywords.rb')
-        self.keywords
+      lazy do
+        require_relative 'viml/keywords'
       end
 
       state :root do
@@ -47,15 +46,14 @@ module Rouge
         rule %r/[absg]:\w+\b/, Name::Variable
         rule %r/\b\w+\b/ do |m|
           name = m[0]
-          keywords = self.class.keywords
 
-          if keywords[:command].include? name
+          if KEYWORDS[:command].include? name
             token Keyword
-          elsif keywords[:function].include? name
-            token Name::Builtin
-          elsif keywords[:option].include? name
-            token Name::Builtin
-          elsif keywords[:auto].include? name
+          elsif (
+            KEYWORDS[:function].include?(name) ||
+            KEYWORDS[:option].include?(name) ||
+            KEYWORDS[:auto].include?(name)
+          )
             token Name::Builtin
           else
             token Text

data/lib/rouge/lexers/vue.rb

@@ -1,10 +1,23 @@
 # frozen_string_literal: true
 
+require_relative 'html'
+
 module Rouge
   module Lexers
-    load_lexer 'html.rb'
-
     class Vue < HTML
+      # HTML extended with Vue template attribute shorthands.
+      # Used as the delegate lexer for <template> content so that directives
+      # like @click and @click.disabled are recognised without polluting the plain
+      # HTML lexer with framework-specific syntax.
+      class TemplateHTML < HTML
+        prepend :tag do
+          rule %r/([@a-zA-Z0-9_:#\[\]()*.-]+\s*)(=)(\s*)/m do
+            groups Name::Attribute, Operator, Text
+            push :attr
+          end
+        end
+      end
+
       desc 'Vue.js single-file components'
       tag 'vue'
       aliases 'vuejs'
@@ -38,7 +51,7 @@ module Rouge
       prepend :root do
         rule %r/(<)(\s*)(template)/ do
           groups Name::Tag, Text, Keyword
-          @lang = HTML
+          @lang = TemplateHTML
           push :template
           push :lang_tag
         end
@@ -59,7 +72,10 @@ module Rouge
       end
 
       prepend :tag do
-        rule %r/[a-zA-Z0-9_:#\[\]()*.-]+\s*=\s*/m, Name::Attribute, :attr
+        rule %r/([a-zA-Z0-9_:#\[\]()*.-]+\s*)(=)(\s*)/m do
+          groups Name::Attribute, Operator, Text
+          push :attr
+        end
       end
 
       state :style do

data/lib/rouge/lexers/xquery.rb

@@ -1,9 +1,10 @@
 # -*- coding: utf-8 -*- #
 # frozen_string_literal: true
 
+require_relative 'xpath'
+
 module Rouge
   module Lexers
-    load_lexer 'xpath.rb'
     class XQuery < XPath
       title 'XQuery'
       desc 'XQuery 3.1: An XML Query Language'

data/lib/rouge/lexers/yaml.rb

@@ -172,12 +172,24 @@ module Rouge
       end
 
       state :block_nodes do
-        # implicit key
+        # implicit unquoted key
         rule %r/([^#,?\[\]{}"'\n]+)(:)(?=\s|$)/ do |m|
           groups Name::Attribute, Punctuation::Indicator
           set_indent m[0], :implicit => true
         end
 
+        # implicit double-quoted key
+        rule %r/("(?:[^\n"]|\\")*")(\s*)(:)(?=\s|$)/ do |m|
+          groups Name::Attribute, Text, Punctuation::Indicator
+          set_indent m[0], :implicit => true
+        end
+
+        # implicit single-quoted key
+        rule %r/('(?:[^\n']|\\')*')(\s*)(:)(?=\s|$)/ do |m|
+          groups Name::Attribute, Text, Punctuation::Indicator
+          set_indent m[0], :implicit => true
+        end
+
         # literal and folded scalars
         rule %r/[\|>][+-]?/ do
           token Punctuation::Indicator

data/lib/rouge/lexers/yang.rb

@@ -13,74 +13,56 @@ module Rouge
       id = /[\w-]+(?=[^\w\-\:])\b/
 
       #Keywords from RFC7950 ; oriented at BNF style
-      def self.top_stmts_keywords
-        @top_stms_keywords ||= Set.new %w(
-          module submodule
-        )
-      end
+      top_keywords = Set.new %w(module submodule)
 
-      def self.module_header_stmts_keywords
-        @module_header_stmts_keywords ||= Set.new %w(
-          belongs-to namespace prefix yang-version
-        )
-      end
+      module_header_keywords = Set.new %w(belongs-to namespace prefix yang-version)
 
-      def self.meta_stmts_keywords
-        @meta_stmts_keywords ||= Set.new %w(
-          contact description organization reference revision
-        )
-      end
+      meta_keywords = Set.new %w(contact description organization reference revision)
 
-      def self.linkage_stmts_keywords
-        @linkage_stmts_keywords ||= Set.new %w(
-          import include revision-date
-        )
-      end
+      linkage_keywords = Set.new %w(import include revision-date)
 
-      def self.body_stmts_keywords
-        @body_stms_keywords ||= Set.new %w(
-          action argument augment deviation extension feature grouping identity
-          if-feature input notification output rpc typedef
-        )
-      end
+      body_keywords = Set.new %w(
+        action argument augment deviation extension feature grouping identity
+        if-feature input notification output rpc typedef
+      )
 
-      def self.data_def_stmts_keywords
-        @data_def_stms_keywords ||= Set.new %w(
-          anydata anyxml case choice config container deviate leaf leaf-list
-          list must presence refine uses when
-        )
-      end
+      data_def_keywords = Set.new %w(
+        anydata anyxml case choice config container deviate leaf leaf-list
+        list must presence refine uses when
+      )
 
-      def self.type_stmts_keywords
-        @type_stmts_keywords ||= Set.new %w(
-          base bit default enum error-app-tag error-message fraction-digits
-          length max-elements min-elements modifier ordered-by path pattern
-          position range require-instance status type units value yin-element
-        )
-      end
+      type_keywords = Set.new %w(
+        base bit default enum error-app-tag error-message fraction-digits
+        length max-elements min-elements modifier ordered-by path pattern
+        position range require-instance status type units value yin-element
+      )
 
-      def self.list_stmts_keywords
-        @list_stmts_keywords ||= Set.new %w(
-          key mandatory unique
-        )
-      end
+      list_keywords = Set.new %w(
+        key mandatory unique
+      )
 
       #RFC7950 other keywords
-      def self.constants_keywords
-        @constants_keywords ||= Set.new %w(
-          add current delete deprecated false invert-match max min
-          not-supported obsolete replace true unbounded user
-        )
-      end
+      CONSTANTS = Set.new %w(
+        add current delete deprecated false invert-match max min
+        not-supported obsolete replace true unbounded user
+      )
 
       #RFC7950 Built-In Types
-      def self.types
-        @types ||= Set.new %w(
-          binary bits boolean decimal64 empty enumeration identityref
-          instance-identifier int16 int32 int64 int8 leafref string uint16
-          uint32 uint64 uint8 union
-        )
-      end
+      TYPES = Set.new %w(
+        binary bits boolean decimal64 empty enumeration identityref
+        instance-identifier int16 int32 int64 int8 leafref string uint16
+        uint32 uint64 uint8 union
+      )
+
+      DECLARATIONS =
+        top_keywords +
+        module_header_keywords +
+        meta_keywords +
+        linkage_keywords +
+        body_keywords +
+        data_def_keywords +
+        type_keywords +
+        list_keywords
 
       state :comment do
         rule %r/[^*\/]/, Comment
@@ -115,25 +97,11 @@ module Rouge
         rule id do |m|
           name = m[0].downcase
 
-          if self.class.top_stmts_keywords.include? name
-            token Keyword::Declaration
-          elsif self.class.module_header_stmts_keywords.include? name
-            token Keyword::Declaration
-          elsif self.class.meta_stmts_keywords.include? name
-            token Keyword::Declaration
-          elsif self.class.linkage_stmts_keywords.include? name
-            token Keyword::Declaration
-          elsif self.class.body_stmts_keywords.include? name
-            token Keyword::Declaration
-          elsif self.class.data_def_stmts_keywords.include? name
-            token Keyword::Declaration
-          elsif self.class.type_stmts_keywords.include? name
-            token Keyword::Declaration
-          elsif self.class.list_stmts_keywords.include? name
+          if DECLARATIONS.include?(name)
             token Keyword::Declaration
-          elsif self.class.types.include? name
+          elsif TYPES.include? name
             token Keyword::Type
-          elsif self.class.constants_keywords.include? name
+          elsif CONSTANTS.include? name
             token Name::Constant
           else
             token Name

data/lib/rouge/lexers/zig.rb

@@ -84,16 +84,16 @@ module Rouge
         rule %r/([.]?)(\s*)(@?#{id})(\s*)([(]?)/ do |m|
           name = m[3]
           t = if self.class.keywords.include? name
-                Keyword
-              elsif self.class.builtins.include? name
-                Name::Builtin
-              elsif !m[1].empty? && !m[5].empty?
-                Name::Function
-              elsif !m[1].empty?
-                Name::Property
-              else
-                Name
-              end
+            Keyword
+          elsif self.class.builtins.include? name
+            Name::Builtin
+          elsif !m[1].empty? && !m[5].empty?
+            Name::Function
+          elsif !m[1].empty?
+            Name::Property
+          else
+            Name
+          end
 
           groups Punctuation, Text, t, Text, Punctuation
         end

data/lib/rouge/plugins/redcarpet.rb

@@ -25,12 +25,13 @@ module Rouge
         end
 
         formatter = rouge_formatter(lexer)
-        formatter.format(lexer.lex(code))
+        formatted = formatter.format(lexer.lex(code))
+        %(<pre class="highlight #{lexer.tag}"><code>#{formatted}</code></pre>)
       end
 
       # override this method for custom formatting behavior
       def rouge_formatter(lexer)
-        Formatters::HTMLLegacy.new(:css_class => "highlight #{lexer.tag}")
+        Rouge::Formatters::HTML.new
       end
     end
   end

data/lib/rouge/regex_lexer.rb

@@ -37,6 +37,12 @@ module Rouge
       end
     end
 
+    # exception class for fallthrough - using an exception is slightly faster
+    # than catch { ... }, but it is not semantically an error that should be
+    # rescued from.
+    class Fallthrough < Exception # rubocop:disable Lint/InheritException
+    end
+
     # A rule is a tuple of a regular expression to test, and a callback
     # to perform if the test succeeds.
     #
@@ -44,11 +50,9 @@ module Rouge
     class Rule
       attr_reader :callback
       attr_reader :re
-      attr_reader :beginning_of_line
       def initialize(re, callback)
         @re = re
         @callback = callback
-        @beginning_of_line = re.source[0] == ?^
       end
 
       def inspect
@@ -277,6 +281,18 @@ module Rouge
       end
     end
 
+    def inspect_details
+      yield ' stack:['
+      first = true
+      stack.each do |state|
+        yield ',' unless first
+        first = false
+        yield state.name
+      end
+
+      yield ']'
+    end
+
     # @private
     def get_state(state_name)
       self.class.get_state(state_name)
@@ -322,7 +338,7 @@ module Rouge
     #
     # @see #step #step (where (2.) is implemented)
     def stream_tokens(str, &b)
-      stream = StringScanner.new(str)
+      stream = StringScanner.new(str, fixed_anchor: true)
 
       @current_stream = stream
       @output_stream  = b
@@ -333,7 +349,7 @@ module Rouge
         if @debug
           puts
           puts "lexer: #{self.class.tag}"
-          puts "stack: #{stack.map(&:name).map(&:to_sym).inspect}"
+          puts "stack: #{stack.map { |s| s.name.to_sym }.inspect}"
           puts "stream: #{stream.peek(20).inspect}"
         end
 
@@ -341,7 +357,7 @@ module Rouge
 
         if !success
           puts "    no match, yielding Error" if @debug
-          b.call(Token::Tokens::Error, stream.getch)
+          yield(Token::Tokens::Error, stream.getch)
         end
       end
     end
@@ -364,18 +380,9 @@ module Rouge
         else
           puts "  trying: #{rule.inspect}" if @debug
 
-          # XXX HACK XXX
-          # StringScanner's implementation of ^ is b0rken.
-          # see http://bugs.ruby-lang.org/issues/7092
-          # TODO: this doesn't cover cases like /(a|^b)/, but it's
-          # the most common, for now...
-          next if rule.beginning_of_line && !stream.beginning_of_line?
-
           if (size = stream.skip(rule.re))
             puts "    got: #{stream[0].inspect}" if @debug
 
-            instance_exec(stream, &rule.callback)
-
             if size.zero?
               @null_steps += 1
               if @null_steps > MAX_NULL_SCANS
@@ -386,6 +393,13 @@ module Rouge
               @null_steps = 0
             end
 
+            begin
+              instance_exec(stream, &rule.callback)
+            rescue Fallthrough
+              stream.unscan
+              next
+            end
+
             return true
           end
         end
@@ -440,10 +454,19 @@ module Rouge
       end
     end
 
+    # Re-lexes the given text (or the most recently matched string if
+    # none is given) with the current lexer.
     def recurse(text=nil)
       delegate(self.class, text)
     end
 
+    # Breaks out of the current rule block and continues to match later
+    # rules, as if the current regex had not matched. Does not affect
+    # the stack.
+    def fallthrough!
+      raise Fallthrough
+    end
+
     # Push a state onto the stack.  If no state name is given and you've
     # passed a block, a state will be dynamically created using the
     # {StateDSL}.

data/lib/rouge/themes/gruvbox.rb

@@ -10,6 +10,7 @@ module Rouge
     class Gruvbox < CSSTheme
       name 'gruvbox'
 
+      # rubocop:disable Naming/ConstantName
       # global Gruvbox colours {{{
       C_dark0_hard = '#1d2021'
       C_dark0 ='#282828'
@@ -122,6 +123,8 @@ module Rouge
         palette orange: C_faded_orange
       end
 
+      # rubocop:enable Naming/ConstantName
+
       dark!
       mode :light
 

data/lib/rouge/themes/thankful_eyes.rb

@@ -60,7 +60,8 @@ module Rouge
             Literal::Date,
             Literal::String::Symbol, :fg => :pink_merengue, :bold => true
       style Literal::String, :fg => :dune, :bold => true
-      style Literal::String::Affix, :fg => :sandy, :bold => true
+      style Literal::String::Affix,
+            Literal::String::Delimiter, :fg => :sandy, :bold => true
       style Literal::String::Escape,
             Literal::String::Char,
             Literal::String::Interpol, :fg => :backlit, :bold => true

data/lib/rouge/token.rb

@@ -61,7 +61,7 @@ module Rouge
 
       def each_token(&b)
         Token.cache.each do |(_, t)|
-          b.call(t)
+          yield(t)
         end
       end
     end

data/lib/rouge/util.rb

@@ -24,7 +24,7 @@ module Rouge
 
     def each(&b)
       keys.each do |k|
-        b.call(k, self[k])
+        yield(k, self[k])
       end
     end
 

data/lib/rouge/version.rb

@@ -3,6 +3,6 @@
 
 module Rouge
   def self.version
-    "4.7.0"
+    "5.1.0"
   end
 end

data/lib/rouge.rb

@@ -3,6 +3,9 @@
 
 # stdlib
 require 'pathname'
+require 'set'
+require 'strscan'
+require 'uri'
 
 # The containing module for Rouge
 module Rouge
@@ -11,7 +14,12 @@ module Rouge
   LIB_DIR = __dir__.freeze
 
   class << self
+    # @deprecated This method of reloading is incompatible with modern Ruby's
+    # expectations around global caching of `require`. It is deprecated with no
+    # replacement - consider reloading the entire process instead with something
+    # like Guard. This method will be removed in rouge 5.0.
     def reload!
+      Kernel::warn "Rouge.reload! is deprecated, with no replacement, and will be removed in 5.0. Use a reloading system like Guard instead."
       Object::send :remove_const, :Rouge
       Kernel::load __FILE__
     end
@@ -36,71 +44,74 @@ module Rouge
       formatter.format(lexer.lex(text), &b)
     end
 
-    # Load a file relative to the `lib/rouge` path.
-    #
-    # @api private
-    def load_file(path)
-      Kernel::load File.join(LIB_DIR, "rouge/#{path}.rb")
-    end
-
-    # Load the lexers in the `lib/rouge/lexers` directory.
-    #
-    # @api private
-    def load_lexers
-      lexer_dir = Pathname.new(LIB_DIR) / "rouge/lexers"
-      Pathname.glob(lexer_dir / '*.rb').each do |f|
-        Lexers.load_lexer(f.relative_path_from(lexer_dir))
-      end
+    def eager_load!
+      Rouge::Lexer.all.each(&:eager_load!)
     end
   end
 end
 
-Rouge.load_file 'version'
-Rouge.load_file 'util'
-Rouge.load_file 'text_analyzer'
-Rouge.load_file 'token'
+require_relative 'rouge/version'
+require_relative 'rouge/util'
+require_relative 'rouge/text_analyzer'
+require_relative 'rouge/token'
 
-Rouge.load_file 'lexer'
-Rouge.load_file 'regex_lexer'
-Rouge.load_file 'template_lexer'
+require_relative 'rouge/lexer'
+require_relative 'rouge/regex_lexer'
+require_relative 'rouge/template_lexer'
 
-Rouge.load_lexers
+# workaround for https://bugs.ruby-lang.org/issues/21870
+# delete this entire begin/end block (or guard it against RUBY_VERSION)
+# when that is fixed.
+begin
+  old_verbose, $VERBOSE = $VERBOSE, false
+  require_relative 'rouge/lexers/xml'
+  require_relative 'rouge/lexers/css'
+  require_relative 'rouge/lexers/html'
+  require_relative 'rouge/lexers/julia'
+ensure
+  $VERBOSE = old_verbose
+end
+
+Dir.glob('rouge/lexers/*.rb', base: __dir__).each do |file|
+  require_relative file
+end
 
-Rouge.load_file 'guesser'
-Rouge.load_file 'guessers/util'
-Rouge.load_file 'guessers/glob_mapping'
-Rouge.load_file 'guessers/modeline'
-Rouge.load_file 'guessers/filename'
-Rouge.load_file 'guessers/mimetype'
-Rouge.load_file 'guessers/source'
-Rouge.load_file 'guessers/disambiguation'
+require_relative 'rouge/guesser'
+require_relative 'rouge/guessers/util'
+require_relative 'rouge/guessers/glob_mapping'
+require_relative 'rouge/guessers/modeline'
+require_relative 'rouge/guessers/filename'
+require_relative 'rouge/guessers/mimetype'
+require_relative 'rouge/guessers/source'
+require_relative 'rouge/guessers/disambiguation'
 
-Rouge.load_file 'formatter'
-Rouge.load_file 'formatters/html'
-Rouge.load_file 'formatters/html_table'
-Rouge.load_file 'formatters/html_pygments'
-Rouge.load_file 'formatters/html_legacy'
-Rouge.load_file 'formatters/html_linewise'
-Rouge.load_file 'formatters/html_line_highlighter'
-Rouge.load_file 'formatters/html_line_table'
-Rouge.load_file 'formatters/html_inline'
-Rouge.load_file 'formatters/terminal256'
-Rouge.load_file 'formatters/terminal_truecolor'
-Rouge.load_file 'formatters/tex'
-Rouge.load_file 'formatters/null'
+require_relative 'rouge/formatter'
+require_relative 'rouge/formatters/html'
+require_relative 'rouge/formatters/html_table'
+require_relative 'rouge/formatters/html_pygments'
+require_relative 'rouge/formatters/html_legacy'
+require_relative 'rouge/formatters/html_linewise'
+require_relative 'rouge/formatters/html_line_highlighter'
+require_relative 'rouge/formatters/html_line_table'
+require_relative 'rouge/formatters/html_inline'
+require_relative 'rouge/formatters/html_debug'
+require_relative 'rouge/formatters/terminal256'
+require_relative 'rouge/formatters/terminal_truecolor'
+require_relative 'rouge/formatters/tex'
+require_relative 'rouge/formatters/null'
 
-Rouge.load_file 'theme'
-Rouge.load_file 'tex_theme_renderer'
-Rouge.load_file 'themes/thankful_eyes'
-Rouge.load_file 'themes/colorful'
-Rouge.load_file 'themes/base16'
-Rouge.load_file 'themes/github'
-Rouge.load_file 'themes/igor_pro'
-Rouge.load_file 'themes/monokai'
-Rouge.load_file 'themes/molokai'
-Rouge.load_file 'themes/monokai_sublime'
-Rouge.load_file 'themes/gruvbox'
-Rouge.load_file 'themes/tulip'
-Rouge.load_file 'themes/pastie'
-Rouge.load_file 'themes/bw'
-Rouge.load_file 'themes/magritte'
+require_relative 'rouge/theme'
+require_relative 'rouge/tex_theme_renderer'
+require_relative 'rouge/themes/thankful_eyes'
+require_relative 'rouge/themes/colorful'
+require_relative 'rouge/themes/base16'
+require_relative 'rouge/themes/github'
+require_relative 'rouge/themes/igor_pro'
+require_relative 'rouge/themes/monokai'
+require_relative 'rouge/themes/molokai'
+require_relative 'rouge/themes/monokai_sublime'
+require_relative 'rouge/themes/gruvbox'
+require_relative 'rouge/themes/tulip'
+require_relative 'rouge/themes/pastie'
+require_relative 'rouge/themes/bw'
+require_relative 'rouge/themes/magritte'