rouge 4.7.0 → 5.0.0

data/lib/rouge/lexers/vue.rb

@@ -1,9 +1,9 @@
 # frozen_string_literal: true
 
+require_relative 'html'
+
 module Rouge
   module Lexers
-    load_lexer 'html.rb'
-
     class Vue < HTML
       desc 'Vue.js single-file components'
       tag 'vue'

data/lib/rouge/lexers/xquery.rb

@@ -1,9 +1,10 @@
 # -*- coding: utf-8 -*- #
 # frozen_string_literal: true
 
+require_relative 'xpath'
+
 module Rouge
   module Lexers
-    load_lexer 'xpath.rb'
     class XQuery < XPath
       title 'XQuery'
       desc 'XQuery 3.1: An XML Query Language'

data/lib/rouge/lexers/yaml.rb

@@ -172,12 +172,24 @@ module Rouge
       end
 
       state :block_nodes do
-        # implicit key
+        # implicit unquoted key
         rule %r/([^#,?\[\]{}"'\n]+)(:)(?=\s|$)/ do |m|
           groups Name::Attribute, Punctuation::Indicator
           set_indent m[0], :implicit => true
         end
 
+        # implicit double-quoted key
+        rule %r/("(?:[^\n"]|\\")*")(\s*)(:)(?=\s|$)/ do |m|
+          groups Name::Attribute, Text, Punctuation::Indicator
+          set_indent m[0], :implicit => true
+        end
+
+        # implicit single-quoted key
+        rule %r/('(?:[^\n']|\\')*')(\s*)(:)(?=\s|$)/ do |m|
+          groups Name::Attribute, Text, Punctuation::Indicator
+          set_indent m[0], :implicit => true
+        end
+
         # literal and folded scalars
         rule %r/[\|>][+-]?/ do
           token Punctuation::Indicator

data/lib/rouge/lexers/yang.rb

@@ -13,74 +13,56 @@ module Rouge
       id = /[\w-]+(?=[^\w\-\:])\b/
 
       #Keywords from RFC7950 ; oriented at BNF style
-      def self.top_stmts_keywords
-        @top_stms_keywords ||= Set.new %w(
-          module submodule
-        )
-      end
+      top_keywords = Set.new %w(module submodule)
 
-      def self.module_header_stmts_keywords
-        @module_header_stmts_keywords ||= Set.new %w(
-          belongs-to namespace prefix yang-version
-        )
-      end
+      module_header_keywords = Set.new %w(belongs-to namespace prefix yang-version)
 
-      def self.meta_stmts_keywords
-        @meta_stmts_keywords ||= Set.new %w(
-          contact description organization reference revision
-        )
-      end
+      meta_keywords = Set.new %w(contact description organization reference revision)
 
-      def self.linkage_stmts_keywords
-        @linkage_stmts_keywords ||= Set.new %w(
-          import include revision-date
-        )
-      end
+      linkage_keywords = Set.new %w(import include revision-date)
 
-      def self.body_stmts_keywords
-        @body_stms_keywords ||= Set.new %w(
-          action argument augment deviation extension feature grouping identity
-          if-feature input notification output rpc typedef
-        )
-      end
+      body_keywords = Set.new %w(
+        action argument augment deviation extension feature grouping identity
+        if-feature input notification output rpc typedef
+      )
 
-      def self.data_def_stmts_keywords
-        @data_def_stms_keywords ||= Set.new %w(
-          anydata anyxml case choice config container deviate leaf leaf-list
-          list must presence refine uses when
-        )
-      end
+      data_def_keywords = Set.new %w(
+        anydata anyxml case choice config container deviate leaf leaf-list
+        list must presence refine uses when
+      )
 
-      def self.type_stmts_keywords
-        @type_stmts_keywords ||= Set.new %w(
-          base bit default enum error-app-tag error-message fraction-digits
-          length max-elements min-elements modifier ordered-by path pattern
-          position range require-instance status type units value yin-element
-        )
-      end
+      type_keywords = Set.new %w(
+        base bit default enum error-app-tag error-message fraction-digits
+        length max-elements min-elements modifier ordered-by path pattern
+        position range require-instance status type units value yin-element
+      )
 
-      def self.list_stmts_keywords
-        @list_stmts_keywords ||= Set.new %w(
-          key mandatory unique
-        )
-      end
+      list_keywords = Set.new %w(
+        key mandatory unique
+      )
 
       #RFC7950 other keywords
-      def self.constants_keywords
-        @constants_keywords ||= Set.new %w(
-          add current delete deprecated false invert-match max min
-          not-supported obsolete replace true unbounded user
-        )
-      end
+      CONSTANTS = Set.new %w(
+        add current delete deprecated false invert-match max min
+        not-supported obsolete replace true unbounded user
+      )
 
       #RFC7950 Built-In Types
-      def self.types
-        @types ||= Set.new %w(
-          binary bits boolean decimal64 empty enumeration identityref
-          instance-identifier int16 int32 int64 int8 leafref string uint16
-          uint32 uint64 uint8 union
-        )
-      end
+      TYPES = Set.new %w(
+        binary bits boolean decimal64 empty enumeration identityref
+        instance-identifier int16 int32 int64 int8 leafref string uint16
+        uint32 uint64 uint8 union
+      )
+
+      DECLARATIONS =
+        top_keywords +
+        module_header_keywords +
+        meta_keywords +
+        linkage_keywords +
+        body_keywords +
+        data_def_keywords +
+        type_keywords +
+        list_keywords
 
       state :comment do
         rule %r/[^*\/]/, Comment
@@ -115,25 +97,11 @@ module Rouge
         rule id do |m|
           name = m[0].downcase
 
-          if self.class.top_stmts_keywords.include? name
-            token Keyword::Declaration
-          elsif self.class.module_header_stmts_keywords.include? name
-            token Keyword::Declaration
-          elsif self.class.meta_stmts_keywords.include? name
-            token Keyword::Declaration
-          elsif self.class.linkage_stmts_keywords.include? name
-            token Keyword::Declaration
-          elsif self.class.body_stmts_keywords.include? name
-            token Keyword::Declaration
-          elsif self.class.data_def_stmts_keywords.include? name
-            token Keyword::Declaration
-          elsif self.class.type_stmts_keywords.include? name
-            token Keyword::Declaration
-          elsif self.class.list_stmts_keywords.include? name
+          if DECLARATIONS.include?(name)
             token Keyword::Declaration
-          elsif self.class.types.include? name
+          elsif TYPES.include? name
             token Keyword::Type
-          elsif self.class.constants_keywords.include? name
+          elsif CONSTANTS.include? name
             token Name::Constant
           else
             token Name

data/lib/rouge/lexers/zig.rb

@@ -84,16 +84,16 @@ module Rouge
         rule %r/([.]?)(\s*)(@?#{id})(\s*)([(]?)/ do |m|
           name = m[3]
           t = if self.class.keywords.include? name
-                Keyword
-              elsif self.class.builtins.include? name
-                Name::Builtin
-              elsif !m[1].empty? && !m[5].empty?
-                Name::Function
-              elsif !m[1].empty?
-                Name::Property
-              else
-                Name
-              end
+            Keyword
+          elsif self.class.builtins.include? name
+            Name::Builtin
+          elsif !m[1].empty? && !m[5].empty?
+            Name::Function
+          elsif !m[1].empty?
+            Name::Property
+          else
+            Name
+          end
 
           groups Punctuation, Text, t, Text, Punctuation
         end

data/lib/rouge/plugins/redcarpet.rb

@@ -25,12 +25,13 @@ module Rouge
         end
 
         formatter = rouge_formatter(lexer)
-        formatter.format(lexer.lex(code))
+        formatted = formatter.format(lexer.lex(code))
+        %(<pre class="highlight #{lexer.tag}"><code>#{formatted}</code></pre>)
       end
 
       # override this method for custom formatting behavior
       def rouge_formatter(lexer)
-        Formatters::HTMLLegacy.new(:css_class => "highlight #{lexer.tag}")
+        Rouge::Formatters::HTML.new
       end
     end
   end

data/lib/rouge/regex_lexer.rb

@@ -37,6 +37,12 @@ module Rouge
       end
     end
 
+    # exception class for fallthrough - using an exception is slightly faster
+    # than catch { ... }, but it is not semantically an error that should be
+    # rescued from.
+    class Fallthrough < Exception # rubocop:disable Lint/InheritException
+    end
+
     # A rule is a tuple of a regular expression to test, and a callback
     # to perform if the test succeeds.
     #
@@ -44,11 +50,9 @@ module Rouge
     class Rule
       attr_reader :callback
       attr_reader :re
-      attr_reader :beginning_of_line
       def initialize(re, callback)
         @re = re
         @callback = callback
-        @beginning_of_line = re.source[0] == ?^
       end
 
       def inspect
@@ -322,7 +326,7 @@ module Rouge
     #
     # @see #step #step (where (2.) is implemented)
     def stream_tokens(str, &b)
-      stream = StringScanner.new(str)
+      stream = StringScanner.new(str, fixed_anchor: true)
 
       @current_stream = stream
       @output_stream  = b
@@ -333,7 +337,7 @@ module Rouge
         if @debug
           puts
           puts "lexer: #{self.class.tag}"
-          puts "stack: #{stack.map(&:name).map(&:to_sym).inspect}"
+          puts "stack: #{stack.map { |s| s.name.to_sym }.inspect}"
           puts "stream: #{stream.peek(20).inspect}"
         end
 
@@ -341,7 +345,7 @@ module Rouge
 
         if !success
           puts "    no match, yielding Error" if @debug
-          b.call(Token::Tokens::Error, stream.getch)
+          yield(Token::Tokens::Error, stream.getch)
         end
       end
     end
@@ -364,18 +368,9 @@ module Rouge
         else
           puts "  trying: #{rule.inspect}" if @debug
 
-          # XXX HACK XXX
-          # StringScanner's implementation of ^ is b0rken.
-          # see http://bugs.ruby-lang.org/issues/7092
-          # TODO: this doesn't cover cases like /(a|^b)/, but it's
-          # the most common, for now...
-          next if rule.beginning_of_line && !stream.beginning_of_line?
-
           if (size = stream.skip(rule.re))
             puts "    got: #{stream[0].inspect}" if @debug
 
-            instance_exec(stream, &rule.callback)
-
             if size.zero?
               @null_steps += 1
               if @null_steps > MAX_NULL_SCANS
@@ -386,6 +381,13 @@ module Rouge
               @null_steps = 0
             end
 
+            begin
+              instance_exec(stream, &rule.callback)
+            rescue Fallthrough
+              stream.unscan
+              next
+            end
+
             return true
           end
         end
@@ -440,10 +442,19 @@ module Rouge
       end
     end
 
+    # Re-lexes the given text (or the most recently matched string if
+    # none is given) with the current lexer.
     def recurse(text=nil)
       delegate(self.class, text)
     end
 
+    # Breaks out of the current rule block and continues to match later
+    # rules, as if the current regex had not matched. Does not affect
+    # the stack.
+    def fallthrough!
+      raise Fallthrough
+    end
+
     # Push a state onto the stack.  If no state name is given and you've
     # passed a block, a state will be dynamically created using the
     # {StateDSL}.

data/lib/rouge/themes/gruvbox.rb

@@ -10,6 +10,7 @@ module Rouge
     class Gruvbox < CSSTheme
       name 'gruvbox'
 
+      # rubocop:disable Naming/ConstantName
       # global Gruvbox colours {{{
       C_dark0_hard = '#1d2021'
       C_dark0 ='#282828'
@@ -122,6 +123,8 @@ module Rouge
         palette orange: C_faded_orange
       end
 
+      # rubocop:enable Naming/ConstantName
+
       dark!
       mode :light
 

data/lib/rouge/token.rb

@@ -61,7 +61,7 @@ module Rouge
 
       def each_token(&b)
         Token.cache.each do |(_, t)|
-          b.call(t)
+          yield(t)
         end
       end
     end

data/lib/rouge/util.rb

@@ -24,7 +24,7 @@ module Rouge
 
     def each(&b)
       keys.each do |k|
-        b.call(k, self[k])
+        yield(k, self[k])
       end
     end
 

data/lib/rouge/version.rb

@@ -3,6 +3,6 @@
 
 module Rouge
   def self.version
-    "4.7.0"
+    "5.0.0"
   end
 end

data/lib/rouge.rb

@@ -3,6 +3,9 @@
 
 # stdlib
 require 'pathname'
+require 'set'
+require 'strscan'
+require 'uri'
 
 # The containing module for Rouge
 module Rouge
@@ -11,7 +14,12 @@ module Rouge
   LIB_DIR = __dir__.freeze
 
   class << self
+    # @deprecated This method of reloading is incompatible with modern Ruby's
+    # expectations around global caching of `require`. It is deprecated with no
+    # replacement - consider reloading the entire process instead with something
+    # like Guard. This method will be removed in rouge 5.0.
     def reload!
+      Kernel::warn "Rouge.reload! is deprecated, with no replacement, and will be removed in 5.0. Use a reloading system like Guard instead."
       Object::send :remove_const, :Rouge
       Kernel::load __FILE__
     end
@@ -36,71 +44,74 @@ module Rouge
       formatter.format(lexer.lex(text), &b)
     end
 
-    # Load a file relative to the `lib/rouge` path.
-    #
-    # @api private
-    def load_file(path)
-      Kernel::load File.join(LIB_DIR, "rouge/#{path}.rb")
-    end
-
-    # Load the lexers in the `lib/rouge/lexers` directory.
-    #
-    # @api private
-    def load_lexers
-      lexer_dir = Pathname.new(LIB_DIR) / "rouge/lexers"
-      Pathname.glob(lexer_dir / '*.rb').each do |f|
-        Lexers.load_lexer(f.relative_path_from(lexer_dir))
-      end
+    def eager_load!
+      Rouge::Lexer.all.each(&:eager_load!)
     end
   end
 end
 
-Rouge.load_file 'version'
-Rouge.load_file 'util'
-Rouge.load_file 'text_analyzer'
-Rouge.load_file 'token'
+require_relative 'rouge/version'
+require_relative 'rouge/util'
+require_relative 'rouge/text_analyzer'
+require_relative 'rouge/token'
 
-Rouge.load_file 'lexer'
-Rouge.load_file 'regex_lexer'
-Rouge.load_file 'template_lexer'
+require_relative 'rouge/lexer'
+require_relative 'rouge/regex_lexer'
+require_relative 'rouge/template_lexer'
 
-Rouge.load_lexers
+# workaround for https://bugs.ruby-lang.org/issues/21870
+# delete this entire begin/end block (or guard it against RUBY_VERSION)
+# when that is fixed.
+begin
+  old_verbose, $VERBOSE = $VERBOSE, false
+  require_relative 'rouge/lexers/xml'
+  require_relative 'rouge/lexers/css'
+  require_relative 'rouge/lexers/html'
+  require_relative 'rouge/lexers/julia'
+ensure
+  $VERBOSE = old_verbose
+end
+
+Dir.glob('rouge/lexers/*.rb', base: __dir__).each do |file|
+  require_relative file
+end
 
-Rouge.load_file 'guesser'
-Rouge.load_file 'guessers/util'
-Rouge.load_file 'guessers/glob_mapping'
-Rouge.load_file 'guessers/modeline'
-Rouge.load_file 'guessers/filename'
-Rouge.load_file 'guessers/mimetype'
-Rouge.load_file 'guessers/source'
-Rouge.load_file 'guessers/disambiguation'
+require_relative 'rouge/guesser'
+require_relative 'rouge/guessers/util'
+require_relative 'rouge/guessers/glob_mapping'
+require_relative 'rouge/guessers/modeline'
+require_relative 'rouge/guessers/filename'
+require_relative 'rouge/guessers/mimetype'
+require_relative 'rouge/guessers/source'
+require_relative 'rouge/guessers/disambiguation'
 
-Rouge.load_file 'formatter'
-Rouge.load_file 'formatters/html'
-Rouge.load_file 'formatters/html_table'
-Rouge.load_file 'formatters/html_pygments'
-Rouge.load_file 'formatters/html_legacy'
-Rouge.load_file 'formatters/html_linewise'
-Rouge.load_file 'formatters/html_line_highlighter'
-Rouge.load_file 'formatters/html_line_table'
-Rouge.load_file 'formatters/html_inline'
-Rouge.load_file 'formatters/terminal256'
-Rouge.load_file 'formatters/terminal_truecolor'
-Rouge.load_file 'formatters/tex'
-Rouge.load_file 'formatters/null'
+require_relative 'rouge/formatter'
+require_relative 'rouge/formatters/html'
+require_relative 'rouge/formatters/html_table'
+require_relative 'rouge/formatters/html_pygments'
+require_relative 'rouge/formatters/html_legacy'
+require_relative 'rouge/formatters/html_linewise'
+require_relative 'rouge/formatters/html_line_highlighter'
+require_relative 'rouge/formatters/html_line_table'
+require_relative 'rouge/formatters/html_inline'
+require_relative 'rouge/formatters/html_debug'
+require_relative 'rouge/formatters/terminal256'
+require_relative 'rouge/formatters/terminal_truecolor'
+require_relative 'rouge/formatters/tex'
+require_relative 'rouge/formatters/null'
 
-Rouge.load_file 'theme'
-Rouge.load_file 'tex_theme_renderer'
-Rouge.load_file 'themes/thankful_eyes'
-Rouge.load_file 'themes/colorful'
-Rouge.load_file 'themes/base16'
-Rouge.load_file 'themes/github'
-Rouge.load_file 'themes/igor_pro'
-Rouge.load_file 'themes/monokai'
-Rouge.load_file 'themes/molokai'
-Rouge.load_file 'themes/monokai_sublime'
-Rouge.load_file 'themes/gruvbox'
-Rouge.load_file 'themes/tulip'
-Rouge.load_file 'themes/pastie'
-Rouge.load_file 'themes/bw'
-Rouge.load_file 'themes/magritte'
+require_relative 'rouge/theme'
+require_relative 'rouge/tex_theme_renderer'
+require_relative 'rouge/themes/thankful_eyes'
+require_relative 'rouge/themes/colorful'
+require_relative 'rouge/themes/base16'
+require_relative 'rouge/themes/github'
+require_relative 'rouge/themes/igor_pro'
+require_relative 'rouge/themes/monokai'
+require_relative 'rouge/themes/molokai'
+require_relative 'rouge/themes/monokai_sublime'
+require_relative 'rouge/themes/gruvbox'
+require_relative 'rouge/themes/tulip'
+require_relative 'rouge/themes/pastie'
+require_relative 'rouge/themes/bw'
+require_relative 'rouge/themes/magritte'

data/lib/rubocop/cop/rouge/no_building_alternation_pattern_in_regexp.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module RuboCop
+  module Cop
+    module Rouge
+      # Checks for the use of `.join('|')` or `Regexp.union` inside
+      # interpolated regular expressions.
+      #
+      # Building alternation patterns by joining arrays or using
+      # `Regexp.union` inside a regexp harms performance — the regex
+      # engine must backtrack through every alternative on each match
+      # attempt. It also risks quoting bugs when values contain regex
+      # metacharacters.
+      #
+      # Prefer a `Set` lookup with a simple regex pattern instead.
+      #
+      # @example
+      #   # bad — joins an array into a huge alternation regex
+      #   KEYWORDS = %w[if else while ...]
+      #   rule %r/\b(#{KEYWORDS.join('|')})\b/, Keyword
+      #
+      #   # bad — Regexp.union inside a regex has the same problem
+      #   rule %r/#{Regexp.union(keywords)}/
+      #
+      #   # good — simple regex + Set lookup
+      #   KEYWORDS = Set.new(%w[if else while ...])
+      #   rule %r/\w+/ do |m|
+      #     if KEYWORDS.include?(m[0])
+      #       token Keyword
+      #     else
+      #       token Name
+      #     end
+      #   end
+      #
+      class NoBuildingAlternationPatternInRegexp < RuboCop::Cop::Base
+        MSG = 'Avoid building alternation patterns inside a regexp. ' \
+              'Use a Set lookup with a simple regex pattern for performance.'
+
+        # @!method join_pipe_call?(node)
+        #   Checks if a node is a `.join('|')` or `.join("|")` call.
+        def_node_matcher :join_pipe_call?, <<~PATTERN
+          (send _ :join (str "|"))
+        PATTERN
+
+        # @!method regexp_union_call?(node)
+        #   Checks if a node is a `Regexp.union(...)` call.
+        def_node_matcher :regexp_union_call?, <<~PATTERN
+          (send (const nil? :Regexp) :union ...)
+        PATTERN
+
+        def on_regexp(node)
+          node.children.each do |child|
+            next unless child.begin_type?
+
+            find_offending_calls(child)
+          end
+        end
+
+        private
+
+        def find_offending_calls(begin_node)
+          begin_node.each_descendant(:send) do |send_node|
+            next unless join_pipe_call?(send_node) || regexp_union_call?(send_node)
+
+            add_offense(send_node)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rubocop/cop/rouge/no_huge_collections.rb

@@ -0,0 +1,37 @@
+module RuboCop
+  module Cop
+    module Rouge
+      class NoHugeCollections < RuboCop::Cop::Base
+        MSG = 'Avoid collections of over 300 elements. ' \
+              'Place those in a lazy-loaded file, ideally ' \
+              'generated by a Rake task from the relevant documentation.'
+
+
+        def_node_matcher :set_literal?, <<~PATTERN
+          (send (const nil? :Set) :[] ...)
+        PATTERN
+
+        RESTRICT_ON_SEND = [:[]]
+        MAX_SIZE = 300
+
+        Set["1", "2", "3"]
+        Set.new %w(1 2 3)
+
+        def on_send(node)
+          return unless set_literal?(node)
+
+          # adjust by 2 for the target and method name
+          if node.children.size - 2 > MAX_SIZE
+            add_offense(node)
+          end
+        end
+
+        def on_array(node)
+          if node.children.size >= MAX_SIZE
+            add_offense(node)
+          end
+        end
+      end
+    end
+  end
+end