rouge 3.3.0 → 3.8.0

data/lib/rouge/formatters/html_line_table.rb

@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Formatters
+    class HTMLLineTable < Formatter
+      tag 'html_line_table'
+
+      # @param [Rouge::Formatters::Formatter] formatter An instance of a
+      #   `Rouge::Formatters::HTML` or `Rouge::Formatters::HTMLInline`
+      # @param [Hash] opts options for HTMLLineTable instance.
+      # @option opts [Integer] :start_line line number to start from. Defaults to `1`.
+      # @option opts [String] :table_class Class name for the table.
+      #   Defaults to `"rouge-line-table"`.
+      # @option opts [String] :line_id  a `sprintf` template for generating an `id`
+      #   attribute for each table row corresponding to current line number.
+      #   Defaults to `"line-%i"`.
+      # @option opts [String] :line_class Class name for each table row.
+      #   Defaults to `"lineno"`.
+      # @option opts [String] :gutter_class Class name for rendered line-number cell.
+      #   Defaults to `"rouge-gutter"`.
+      # @option opts [String] :code_class Class name for rendered code cell.
+      #   Defaults to `"rouge-code"`.
+      def initialize(formatter, opts={})
+        @formatter    = formatter
+        @start_line   = opts.fetch :start_line,   1
+        @table_class  = opts.fetch :table_class,  'rouge-line-table'
+        @gutter_class = opts.fetch :gutter_class, 'rouge-gutter'
+        @code_class   = opts.fetch :code_class,   'rouge-code'
+        @line_class   = opts.fetch :line_class,   'lineno'
+        @line_id      = opts.fetch :line_id,      'line-%i'
+      end
+
+      def stream(tokens, &b)
+        lineno = @start_line - 1
+        buffer = [%(<table class="#@table_class"><tbody>)]
+        token_lines(tokens) do |line_tokens|
+          lineno += 1
+          buffer << %(<tr id="#{sprintf @line_id, lineno}" class="#@line_class">)
+          buffer << %(<td class="#@gutter_class gl" )
+          buffer << %(style="-moz-user-select: none;-ms-user-select: none;)
+          buffer << %(-webkit-user-select: none;user-select: none;">)
+          buffer << %(<pre>#{lineno}</pre></td>)
+          buffer << %(<td class="#@code_class"><pre>)
+          @formatter.stream(line_tokens) { |formatted| buffer << formatted }
+          buffer << "\n</pre></td></tr>"
+        end
+        buffer << %(</tbody></table>)
+        yield buffer.join
+      end
+    end
+  end
+end

data/lib/rouge/formatters/html_linewise.rb

@@ -6,23 +6,18 @@ module Rouge
     class HTMLLinewise < Formatter
       def initialize(formatter, opts={})
         @formatter = formatter
+        @tag_name = opts.fetch(:tag_name, 'div')
         @class_format = opts.fetch(:class, 'line-%i')
       end
 
       def stream(tokens, &b)
-        token_lines(tokens) do |line|
-          yield "<div class=#{next_line_class}>"
-          line.each do |tok, val|
-            yield @formatter.span(tok, val)
-          end
-          yield '</div>'
+        lineno = 0
+        token_lines(tokens) do |line_tokens|
+          yield %(<#{@tag_name} class="#{sprintf @class_format, lineno += 1}">)
+          @formatter.stream(line_tokens) {|formatted| yield formatted }
+          yield %(\n</#{@tag_name}>)
         end
       end
-
-      def next_line_class
-        @lineno ||= 0
-        sprintf(@class_format, @lineno += 1).inspect
-      end
     end
   end
 end

data/lib/rouge/formatters/html_table.rb

@@ -16,46 +16,35 @@ module Rouge
       end
 
       def style(scope)
-        yield "#{scope} .rouge-table { border-spacing: 0 }"
-        yield "#{scope} .rouge-gutter { text-align: right }"
+        yield %(#{scope} .rouge-table { border-spacing: 0 })
+        yield %(#{scope} .rouge-gutter { text-align: right })
       end
 
       def stream(tokens, &b)
-        num_lines = 0
-        last_val = ''
-        formatted = String.new('')
-
-        tokens.each do |tok, val|
-          last_val = val
-          num_lines += val.scan(/\n/).size
-          formatted << @inner.span(tok, val)
-        end
-
-        # add an extra line for non-newline-terminated strings
-        if last_val[-1] != "\n"
+        last_val = nil
+        num_lines = tokens.reduce(0) {|count, (_, val)| count + (last_val = val).count(?\n) }
+        formatted = @inner.format(tokens)
+        unless last_val && last_val.end_with?(?\n)
           num_lines += 1
-          @inner.span(Token::Tokens::Text::Whitespace, "\n") { |str| formatted << str }
+          formatted << ?\n
         end
 
         # generate a string of newline-separated line numbers for the gutter>
-        formatted_line_numbers = (@start_line..num_lines+@start_line-1).map do |i|
-          sprintf("#{@line_format}", i) << "\n"
-        end.join('')
-
-        numbers = %(<pre class="lineno">#{formatted_line_numbers}</pre>)
-
-        yield %(<table class="#@table_class"><tbody><tr>)
+        formatted_line_numbers = (@start_line..(@start_line + num_lines - 1)).map do |i|
+          sprintf(@line_format, i)
+        end.join(?\n) << ?\n
 
+        buffer = [%(<table class="#@table_class"><tbody><tr>)]
         # the "gl" class applies the style for Generic.Lineno
-        yield %(<td class="#@gutter_class gl">)
-        yield numbers
-        yield '</td>'
-
-        yield %(<td class="#@code_class"><pre>)
-        yield formatted
-        yield '</pre></td>'
-
-        yield "</tr></tbody></table>"
+        buffer << %(<td class="#@gutter_class gl">)
+        buffer << %(<pre class="lineno">#{formatted_line_numbers}</pre>)
+        buffer << '</td>'
+        buffer << %(<td class="#@code_class"><pre>)
+        buffer << formatted
+        buffer << '</pre></td>'
+        buffer << '</tr></tbody></table>'
+
+        yield buffer.join
       end
     end
   end

data/lib/rouge/formatters/terminal256.rb

@@ -159,6 +159,7 @@ module Rouge
 
     # private
       def escape_sequence(token)
+        return '' if escape?(token)
         @escape_sequences ||= {}
         @escape_sequences[token.qualname] ||=
           EscapeSequence.new(get_style(token))

data/lib/rouge/formatters/tex.rb

@@ -0,0 +1,92 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Formatters
+    class Tex < Formatter
+      tag 'tex'
+
+      # A map of TeX escape characters.
+      # Newlines are handled specially by using #token_lines
+      # spaces are preserved as long as they aren't at the beginning
+      # of a line. see #tag_first for our initial-space strategy
+      ESCAPE = {
+        '&' => '\&',
+        '%' => '\%',
+        '$' => '\$',
+        '#' => '\#',
+        '_' => '\_',
+        '{' => '\{',
+        '}' => '\}',
+        '~' => '{\textasciitilde}',
+        '^' => '{\textasciicircum}',
+        '|' => '{\textbar}',
+        '\\' => '{\textbackslash}',
+        '`' => '{\textasciigrave}',
+        "'" => "'{}",
+        '"' => '"{}',
+        "\t" => '{\tab}',
+      }
+
+      ESCAPE_REGEX = /[#{ESCAPE.keys.map(&Regexp.method(:escape)).join}]/om
+
+      def initialize(opts={})
+        @prefix = opts.fetch(:prefix) { 'RG' }
+      end
+
+      def escape_tex(str)
+        str.gsub(ESCAPE_REGEX, ESCAPE)
+      end
+
+      def stream(tokens, &b)
+        # surround the output with \begin{RG*}...\end{RG*}
+        yield "\\begin{#{@prefix}*}%\n"
+
+        # we strip the newline off the last line to avoid
+        # an extra line being rendered. we do this by yielding
+        # the \newline tag *before* every line group except
+        # the first.
+        first = true
+
+        token_lines tokens do |line|
+          if first
+            first = false
+          else
+            yield "\\newline%\n"
+          end
+
+          render_line(line, &b)
+        end
+
+        yield "%\n\\end{#{@prefix}*}%\n"
+      end
+
+      def render_line(line, &b)
+        line.each do |(tok, val)|
+          hphantom_tag(tok, val, &b)
+        end
+      end
+
+      # Special handling for leading spaces, since they may be gobbled
+      # by a previous command.  We replace all initial spaces with
+      # \hphantom{xxxx}, which renders an empty space equal to the size
+      # of the x's.
+      def hphantom_tag(tok, val)
+        leading = nil
+        val.sub!(/^[ ]+/) { leading = $&.size; '' }
+        yield "\\hphantom{#{'x' * leading}}" if leading
+        yield tag(tok, val) unless val.empty?
+      end
+
+      def tag(tok, val)
+        if escape?(tok)
+          val
+        elsif tok == Token::Tokens::Text
+          escape_tex(val)
+        else
+          "\\#@prefix{#{tok.shortname}}{#{escape_tex(val)}}"
+        end
+      end
+    end
+  end
+end

data/lib/rouge/guessers/disambiguation.rb

@@ -75,6 +75,8 @@ module Rouge
       disambiguate '*.h' do
         next ObjectiveC if matches?(/@(end|implementation|protocol|property)\b/)
         next ObjectiveC if contains?('@"')
+        next Cpp if matches?(/^\s*(?:catch|class|constexpr|namespace|private|
+                                   protected|public|template|throw|try|using)\b/x)
 
         C
       end
@@ -86,7 +88,11 @@ module Rouge
         next Mathematica if contains?('(*')
         next Mathematica if contains?(':=')
 
+        next Mason if matches?(/<%(def|method|text|doc|args|flags|attr|init|once|shared|perl|cleanup|filter)([^>]*)(>)/)
+
         next Matlab if matches?(/^\s*?%/)
+
+        next Mason if matches? %r!(</?%|<&)!
       end
 
       disambiguate '*.php' do
@@ -101,6 +107,25 @@ module Rouge
 
         Cpp
       end
+
+      disambiguate '*.plist' do
+        next XML if matches?(/\A<\?xml\b/)
+
+        Plist
+      end
+
+      disambiguate '*.sc' do
+        next Python if matches?(/^#/)
+        next SuperCollider if matches?(/(?:^~|;$)/)
+
+        next Python
+      end
+
+      disambiguate 'Messages' do
+        next MsgTrans if matches?(/^[^\s:]+:[^\s:]+/)
+
+        next PlainText
+      end
     end
   end
 end

data/lib/rouge/guessers/glob_mapping.rb

@@ -32,7 +32,7 @@ module Rouge
         basename = File.basename(filename)
 
         collect_best(lexers) do |lexer|
-          score = (@glob_map[lexer.name] || []).map do |pattern|
+          (@glob_map[lexer.name] || []).map do |pattern|
             if test_glob(pattern, basename)
               # specificity is better the fewer wildcards there are
               -pattern.scan(/[*?\[]/).size

data/lib/rouge/guessers/modeline.rb

@@ -37,7 +37,7 @@ module Rouge
 
         matches = MODELINES.map { |re| re.match(search_space) }.compact
         return lexers unless matches.any?
-        
+
         match_set = Set.new(matches.map { |m| m[1] })
         lexers.select { |l| match_set.include?(l.tag) || l.aliases.any? { |a| match_set.include?(a) } }
       end

data/lib/rouge/guessers/source.rb

@@ -22,7 +22,7 @@ module Rouge
         source_text = TextAnalyzer.new(source_text)
 
         collect_best(lexers) do |lexer|
-          next unless lexer.methods(false).include? :detect?
+          next unless lexer.detectable?
           lexer.detect?(source_text) ? 1 : nil
         end
       end

data/lib/rouge/lexer.rb

@@ -23,6 +23,14 @@ module Rouge
         new(opts).lex(stream, &b)
       end
 
+      # In case #continue_lex is called statically, we simply
+      # begin a new lex from the beginning, since there is no state.
+      #
+      # @see #continue_lex
+      def continue_lex(*a, &b)
+        lex(*a, &b)
+      end
+
       # Given a name in string, return the correct lexer class.
       # @param [String] name
       # @return [Class<Rouge::Lexer>,nil]
@@ -106,7 +114,7 @@ module Rouge
       def demo_file(arg=:absent)
         return @demo_file = Pathname.new(arg) unless arg == :absent
 
-        @demo_file = Pathname.new(__FILE__).dirname.join('demos', tag)
+        @demo_file = Pathname.new(File.join(__dir__, 'demos', tag))
       end
 
       # Specify or get a small demo string for this lexer
@@ -118,7 +126,7 @@ module Rouge
 
       # @return a list of all lexers.
       def all
-        registry.values.uniq
+        @all ||= registry.values.uniq
       end
 
       # Guess which lexer to use based on a hash of info.
@@ -188,16 +196,24 @@ module Rouge
       end
 
       def disable_debug!
-        @debug_enabled = false
+        remove_instance_variable :@debug_enabled if defined? @debug_enabled
       end
 
       def debug_enabled?
-        !!@debug_enabled
+        (defined? @debug_enabled) ? true : false
+      end
+
+      # Determine if a lexer has a method named +:detect?+ defined in its
+      # singleton class.
+      def detectable?
+        @detectable ||= methods(false).include?(:detect?)
       end
 
     protected
       # @private
       def register(name, lexer)
+        # reset an existing list of lexers
+        @all = nil if defined?(@all)
         registry[name.to_s] = lexer
       end
 
@@ -236,6 +252,13 @@ module Rouge
 
       # Specify a list of filename globs associated with this lexer.
       #
+      # If a filename glob is associated with more than one lexer, this can
+      # cause a Guesser::Ambiguous error to be raised in various guessing
+      # methods. These errors can be avoided by disambiguation. Filename globs
+      # are disambiguated in one of two ways. Either the lexer will define a
+      # `self.detect?` method (intended for use with shebangs and doctypes) or a
+      # manual rule will be specified in Guessers::Disambiguation.
+      #
       # @example
       #   class Ruby < Lexer
       #     filenames '*.rb', '*.ruby', 'Gemfile', 'Rakefile'
@@ -256,7 +279,9 @@ module Rouge
 
       # @private
       def assert_utf8!(str)
-        return if %w(US-ASCII UTF-8 ASCII-8BIT).include? str.encoding.name
+        encoding = str.encoding.name
+        return if encoding == 'US-ASCII' || encoding == 'UTF-8' || encoding == 'ASCII-8BIT'
+
         raise EncodingError.new(
           "Bad encoding: #{str.encoding.names.join(',')}. " +
           "Please convert your string to UTF-8."
@@ -285,7 +310,7 @@ module Rouge
       @options = {}
       opts.each { |k, v| @options[k.to_s] = v }
 
-      @debug = Lexer.debug_enabled? && bool_option(:debug)
+      @debug = Lexer.debug_enabled? && bool_option('debug')
     end
 
     def as_bool(val)
@@ -340,8 +365,10 @@ module Rouge
     end
 
     def bool_option(name, &default)
-      if @options.key?(name.to_s)
-        as_bool(@options[name.to_s])
+      name_str = name.to_s
+
+      if @options.key?(name_str)
+        as_bool(@options[name_str])
       else
         default ? default.call : false
       end
@@ -393,12 +420,37 @@ module Rouge
     #
     # @option opts :continue
     #   Continue the lex from the previous state (i.e. don't call #reset!)
-    def lex(string, opts={}, &b)
-      return enum_for(:lex, string, opts) unless block_given?
+    #
+    # @note The use of :continue => true has been deprecated. A warning is
+    #       issued if run with `$VERBOSE` set to true.
+    #
+    # @note The use of arbitrary `opts` has never been supported, but we
+    #       previously ignored them with no error. We now warn unconditionally.
+    def lex(string, opts=nil, &b)
+      if opts
+        if (opts.keys - [:continue]).size > 0
+          # improper use of options hash
+          warn('Improper use of Lexer#lex - this method does not receive options.' +
+               ' This will become an error in a future version.')
+        end
+
+        if opts[:continue]
+          warn '`lex :continue => true` is deprecated, please use #continue_lex instead'
+          return continue_lex(string, &b)
+        end
+      end
+
+      return enum_for(:lex, string) unless block_given?
 
       Lexer.assert_utf8!(string)
+      reset!
+
+      continue_lex(string, &b)
+    end
 
-      reset! unless opts[:continue]
+    # Continue the lex from the the current state without resetting
+    def continue_lex(string, &b)
+      return enum_for(:continue_lex, string, &b) unless block_given?
 
       # consolidate consecutive tokens of the same type
       last_token = nil
@@ -454,9 +506,7 @@ module Rouge
     def self.load_lexer(relpath)
       return if @_loaded_lexers.key?(relpath)
       @_loaded_lexers[relpath] = true
-
-      root = Pathname.new(__FILE__).dirname.join('lexers')
-      load root.join(relpath)
+      load File.join(__dir__, 'lexers', relpath)
     end
   end
 end