rouge 3.2.0 → 3.18.0

data/lib/rouge/formatters/tex.rb

@@ -0,0 +1,92 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Formatters
+    class Tex < Formatter
+      tag 'tex'
+
+      # A map of TeX escape characters.
+      # Newlines are handled specially by using #token_lines
+      # spaces are preserved as long as they aren't at the beginning
+      # of a line. see #tag_first for our initial-space strategy
+      ESCAPE = {
+        '&' => '\&',
+        '%' => '\%',
+        '$' => '\$',
+        '#' => '\#',
+        '_' => '\_',
+        '{' => '\{',
+        '}' => '\}',
+        '~' => '{\textasciitilde}',
+        '^' => '{\textasciicircum}',
+        '|' => '{\textbar}',
+        '\\' => '{\textbackslash}',
+        '`' => '{\textasciigrave}',
+        "'" => "'{}",
+        '"' => '"{}',
+        "\t" => '{\tab}',
+      }
+
+      ESCAPE_REGEX = /[#{ESCAPE.keys.map(&Regexp.method(:escape)).join}]/om
+
+      def initialize(opts={})
+        @prefix = opts.fetch(:prefix) { 'RG' }
+      end
+
+      def escape_tex(str)
+        str.gsub(ESCAPE_REGEX, ESCAPE)
+      end
+
+      def stream(tokens, &b)
+        # surround the output with \begin{RG*}...\end{RG*}
+        yield "\\begin{#{@prefix}*}%\n"
+
+        # we strip the newline off the last line to avoid
+        # an extra line being rendered. we do this by yielding
+        # the \newline tag *before* every line group except
+        # the first.
+        first = true
+
+        token_lines tokens do |line|
+          if first
+            first = false
+          else
+            yield "\\newline%\n"
+          end
+
+          render_line(line, &b)
+        end
+
+        yield "%\n\\end{#{@prefix}*}%\n"
+      end
+
+      def render_line(line, &b)
+        line.each do |(tok, val)|
+          hphantom_tag(tok, val, &b)
+        end
+      end
+
+      # Special handling for leading spaces, since they may be gobbled
+      # by a previous command.  We replace all initial spaces with
+      # \hphantom{xxxx}, which renders an empty space equal to the size
+      # of the x's.
+      def hphantom_tag(tok, val)
+        leading = nil
+        val.sub!(/^[ ]+/) { leading = $&.size; '' }
+        yield "\\hphantom{#{'x' * leading}}" if leading
+        yield tag(tok, val) unless val.empty?
+      end
+
+      def tag(tok, val)
+        if escape?(tok)
+          val
+        elsif tok == Token::Tokens::Text
+          escape_tex(val)
+        else
+          "\\#@prefix{#{tok.shortname}}{#{escape_tex(val)}}"
+        end
+      end
+    end
+  end
+end

data/lib/rouge/guesser.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rouge
   class Guesser
     class Ambiguous < StandardError

data/lib/rouge/guessers/disambiguation.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rouge
   module Guessers
     class Disambiguation < Guesser
@@ -73,6 +75,8 @@ module Rouge
       disambiguate '*.h' do
         next ObjectiveC if matches?(/@(end|implementation|protocol|property)\b/)
         next ObjectiveC if contains?('@"')
+        next Cpp if matches?(/^\s*(?:catch|class|constexpr|namespace|private|
+                                   protected|public|template|throw|try|using)\b/x)
 
         C
       end
@@ -81,7 +85,14 @@ module Rouge
         next ObjectiveC if matches?(/@(end|implementation|protocol|property)\b/)
         next ObjectiveC if contains?('@"')
 
+        next Mathematica if contains?('(*')
+        next Mathematica if contains?(':=')
+
+        next Mason if matches?(/<%(def|method|text|doc|args|flags|attr|init|once|shared|perl|cleanup|filter)([^>]*)(>)/)
+
         next Matlab if matches?(/^\s*?%/)
+
+        next Mason if matches? %r!(</?%|<&)!
       end
 
       disambiguate '*.php' do
@@ -96,6 +107,30 @@ module Rouge
 
         Cpp
       end
+
+      disambiguate '*.plist' do
+        next XML if matches?(/\A<\?xml\b/)
+
+        Plist
+      end
+
+      disambiguate '*.sc' do
+        next Python if matches?(/^#/)
+        next SuperCollider if matches?(/(?:^~|;$)/)
+
+        next Python
+      end
+
+      disambiguate 'Messages' do
+        next MsgTrans if matches?(/^[^\s:]+:[^\s:]+/)
+
+        next PlainText
+      end
+
+      disambiguate '*.cls' do
+        next TeX if matches?(/\A\s*(?:\\|%)/)
+        next Apex
+      end
     end
   end
 end

data/lib/rouge/guessers/filename.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rouge
   module Guessers
     class Filename < Guesser

data/lib/rouge/guessers/glob_mapping.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rouge
   module Guessers
     # This class allows for custom behavior
@@ -30,7 +32,7 @@ module Rouge
         basename = File.basename(filename)
 
         collect_best(lexers) do |lexer|
-          score = (@glob_map[lexer.name] || []).map do |pattern|
+          (@glob_map[lexer.name] || []).map do |pattern|
             if test_glob(pattern, basename)
               # specificity is better the fewer wildcards there are
               -pattern.scan(/[*?\[]/).size

data/lib/rouge/guessers/mimetype.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rouge
   module Guessers
     class Mimetype < Guesser

data/lib/rouge/guessers/modeline.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rouge
   module Guessers
     class Modeline < Guesser
@@ -35,7 +37,7 @@ module Rouge
 
         matches = MODELINES.map { |re| re.match(search_space) }.compact
         return lexers unless matches.any?
-        
+
         match_set = Set.new(matches.map { |m| m[1] })
         lexers.select { |l| match_set.include?(l.tag) || l.aliases.any? { |a| match_set.include?(a) } }
       end

data/lib/rouge/guessers/source.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rouge
   module Guessers
     class Source < Guesser
@@ -20,7 +22,7 @@ module Rouge
         source_text = TextAnalyzer.new(source_text)
 
         collect_best(lexers) do |lexer|
-          next unless lexer.methods(false).include? :detect?
+          next unless lexer.detectable?
           lexer.detect?(source_text) ? 1 : nil
         end
       end

data/lib/rouge/guessers/util.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rouge
   module Guessers
     module Util

data/lib/rouge/lexer.rb

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*- #
+# frozen_string_literal: true
 
 # stdlib
 require 'strscan'
@@ -22,6 +23,14 @@ module Rouge
         new(opts).lex(stream, &b)
       end
 
+      # In case #continue_lex is called statically, we simply
+      # begin a new lex from the beginning, since there is no state.
+      #
+      # @see #continue_lex
+      def continue_lex(*a, &b)
+        lex(*a, &b)
+      end
+
       # Given a name in string, return the correct lexer class.
       # @param [String] name
       # @return [Class<Rouge::Lexer>,nil]
@@ -40,6 +49,9 @@ module Rouge
       #
       #     Lexer.find_fancy('guess', "#!/bin/bash\necho Hello, world")
       #
+      #   If the code matches more than one lexer then Guesser::Ambiguous
+      #   is raised.
+      #
       # This is used in the Redcarpet plugin as well as Rouge's own
       # markdown lexer for highlighting internal code blocks.
       #
@@ -105,7 +117,7 @@ module Rouge
       def demo_file(arg=:absent)
         return @demo_file = Pathname.new(arg) unless arg == :absent
 
-        @demo_file = Pathname.new(__FILE__).dirname.join('demos', tag)
+        @demo_file = Pathname.new(File.join(__dir__, 'demos', tag))
       end
 
       # Specify or get a small demo string for this lexer
@@ -117,7 +129,7 @@ module Rouge
 
       # @return a list of all lexers.
       def all
-        registry.values.uniq
+        @all ||= registry.values.uniq
       end
 
       # Guess which lexer to use based on a hash of info.
@@ -187,16 +199,25 @@ module Rouge
       end
 
       def disable_debug!
-        @debug_enabled = false
+        remove_instance_variable :@debug_enabled if defined? @debug_enabled
       end
 
       def debug_enabled?
-        !!@debug_enabled
+        (defined? @debug_enabled) ? true : false
+      end
+
+      # Determine if a lexer has a method named +:detect?+ defined in its
+      # singleton class.
+      def detectable?
+        return @detectable if defined?(@detectable)
+        @detectable = singleton_methods(false).include?(:detect?)
       end
 
     protected
       # @private
       def register(name, lexer)
+        # reset an existing list of lexers
+        @all = nil if defined?(@all)
         registry[name.to_s] = lexer
       end
 
@@ -235,6 +256,13 @@ module Rouge
 
       # Specify a list of filename globs associated with this lexer.
       #
+      # If a filename glob is associated with more than one lexer, this can
+      # cause a Guesser::Ambiguous error to be raised in various guessing
+      # methods. These errors can be avoided by disambiguation. Filename globs
+      # are disambiguated in one of two ways. Either the lexer will define a
+      # `self.detect?` method (intended for use with shebangs and doctypes) or a
+      # manual rule will be specified in Guessers::Disambiguation.
+      #
       # @example
       #   class Ruby < Lexer
       #     filenames '*.rb', '*.ruby', 'Gemfile', 'Rakefile'
@@ -255,7 +283,9 @@ module Rouge
 
       # @private
       def assert_utf8!(str)
-        return if %w(US-ASCII UTF-8 ASCII-8BIT).include? str.encoding.name
+        encoding = str.encoding.name
+        return if encoding == 'US-ASCII' || encoding == 'UTF-8' || encoding == 'ASCII-8BIT'
+
         raise EncodingError.new(
           "Bad encoding: #{str.encoding.names.join(',')}. " +
           "Please convert your string to UTF-8."
@@ -284,12 +314,12 @@ module Rouge
       @options = {}
       opts.each { |k, v| @options[k.to_s] = v }
 
-      @debug = Lexer.debug_enabled? && bool_option(:debug)
+      @debug = Lexer.debug_enabled? && bool_option('debug')
     end
 
     def as_bool(val)
       case val
-      when nil, false, 0, '0', 'off'
+      when nil, false, 0, '0', 'false', 'off'
         false
       when Array
         val.empty? ? true : as_bool(val.last)
@@ -339,8 +369,10 @@ module Rouge
     end
 
     def bool_option(name, &default)
-      if @options.key?(name.to_s)
-        as_bool(@options[name.to_s])
+      name_str = name.to_s
+
+      if @options.key?(name_str)
+        as_bool(@options[name_str])
       else
         default ? default.call : false
       end
@@ -392,12 +424,37 @@ module Rouge
     #
     # @option opts :continue
     #   Continue the lex from the previous state (i.e. don't call #reset!)
-    def lex(string, opts={}, &b)
-      return enum_for(:lex, string, opts) unless block_given?
+    #
+    # @note The use of :continue => true has been deprecated. A warning is
+    #       issued if run with `$VERBOSE` set to true.
+    #
+    # @note The use of arbitrary `opts` has never been supported, but we
+    #       previously ignored them with no error. We now warn unconditionally.
+    def lex(string, opts=nil, &b)
+      if opts
+        if (opts.keys - [:continue]).size > 0
+          # improper use of options hash
+          warn('Improper use of Lexer#lex - this method does not receive options.' +
+               ' This will become an error in a future version.')
+        end
+
+        if opts[:continue]
+          warn '`lex :continue => true` is deprecated, please use #continue_lex instead'
+          return continue_lex(string, &b)
+        end
+      end
+
+      return enum_for(:lex, string) unless block_given?
 
       Lexer.assert_utf8!(string)
+      reset!
+
+      continue_lex(string, &b)
+    end
 
-      reset! unless opts[:continue]
+    # Continue the lex from the the current state without resetting
+    def continue_lex(string, &b)
+      return enum_for(:continue_lex, string, &b) unless block_given?
 
       # consolidate consecutive tokens of the same type
       last_token = nil
@@ -448,14 +505,13 @@ module Rouge
   end
 
   module Lexers
+    BASE_DIR = "#{__dir__}/lexers".freeze
     @_loaded_lexers = {}
 
     def self.load_lexer(relpath)
       return if @_loaded_lexers.key?(relpath)
       @_loaded_lexers[relpath] = true
-
-      root = Pathname.new(__FILE__).dirname.join('lexers')
-      load root.join(relpath)
+      load File.join(BASE_DIR, relpath)
     end
   end
 end

data/lib/rouge/lexers/abap.rb

@@ -1,4 +1,6 @@
 # -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
 # ABAP elements taken from http://help.sap.com/abapdocu_750/en/index.htm?file=abapdo.htm
 
 module Rouge
@@ -180,16 +182,16 @@ module Rouge
       end
 
       state :root do
-        rule /\s+/m, Text
+        rule %r/\s+/m, Text
 
-        rule /".*/, Comment::Single
+        rule %r/".*/, Comment::Single
         rule %r(^\*.*), Comment::Multiline
-        rule /\d+/, Num::Integer
-        rule /('|`)/, Str::Single, :single_string
-        rule /[\[\]\(\)\{\}\[\]\.,:\|]/, Punctuation
+        rule %r/\d+/, Num::Integer
+        rule %r/('|`)/, Str::Single, :single_string
+        rule %r/[\[\]\(\)\{\}\.,:\|]/, Punctuation
 
         # builtins / new ABAP 7.40 keywords (@DATA(), ...)
-        rule /(->|=>)?([A-Za-z][A-Za-z0-9_\-]*)(\()/ do |m|
+        rule %r/(->|=>)?([A-Za-z][A-Za-z0-9_\-]*)(\()/ do |m|
           if m[1] != ''
             token Operator, m[1]
           end
@@ -205,7 +207,7 @@ module Rouge
         end
 
         # keywords, types and normal text
-        rule /\w[\w\d]*/ do |m|
+        rule %r/\w\w*/ do |m|
           if self.class.keywords.include? m[0].upcase
             token Keyword
           elsif self.class.types.include? m[0].downcase
@@ -227,10 +229,10 @@ module Rouge
       end
 
       state :single_string do
-        rule /\\./, Str::Escape
-        rule /(''|``)/, Str::Escape
-        rule /['`]/, Str::Single, :pop!
-        rule /[^\\'`]+/, Str::Single
+        rule %r/\\./, Str::Escape
+        rule %r/(''|``)/, Str::Escape
+        rule %r/['`]/, Str::Single, :pop!
+        rule %r/[^\\'`]+/, Str::Single
       end
 
     end