rouge 3.3.0 → 3.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 48cc6c6c6d8f17128a0b1347b6871a2553f5742d
-  data.tar.gz: 7f0d0d895becf6c8a6f52c9e029ea4d1ccceb861
+SHA256:
+  metadata.gz: 5794858ca2a3090f6302e34bf98115e3dd56ad8b7abe30151c304b26ff414547
+  data.tar.gz: e633c0de8ab7553b7873e75108a9fcd233b438fb93a19d5f317a8c3b83faf844
 SHA512:
-  metadata.gz: 073ba0c6ed9e932fd4ab069b99b2cef3d7efcb49dd7841caa71d48ce2faf5cf645eeba1937f56e831c2856ec88f23236ef7a9795e2a453f22c6a22adc36cf47e
-  data.tar.gz: 68e38b4636dc835270f420b884f3f71ceb2bc87d41cc0ed8d811e03e6cd28d6357977911f8e8df3e95f924fa8ddda710af1cc384fd54b7c5eb1aba61fa97438b
+  metadata.gz: 11f8425d849b2f5f0307942c47b99a7504cf6c560d5ae8d87371b901d44ec610cd91b616c2eb6efae6bc0ce3064e0e336b83d2c2393461075f1d14498e239fc1
+  data.tar.gz: d0d07ed86419b3c3b1043b1ff5fcfce7fce1da4fb1a1827edf861603e8f99b79e16b3aa4f2e0f4471eeafce8f79ff93b41cdb79d936afc68302af99fe7e82023

data/Gemfile

@@ -4,17 +4,25 @@ source 'http://rubygems.org'
 
 gemspec
 
-gem 'bundler', '~> 1.15'
 gem 'rake'
 
 gem 'minitest', '>= 5.0'
 gem 'minitest-power_assert'
 
+gem 'parallel', '~> 1.13.0' if RUBY_VERSION < '2.2.0'
 gem 'rubocop', '~> 0.49.1'
 
 # don't try to install redcarpet under jruby
 gem 'redcarpet', :platforms => :ruby
 
+# Profiling
+if RUBY_VERSION >= '2.3.0'
+  gem 'memory_profiler', :require => false
+end
+
+# Needed for a Rake task
+gem 'git'
+
 group :development do
   gem 'pry'
 
@@ -23,6 +31,10 @@ group :development do
   gem 'github-markup'
 
   # for visual tests
-  gem 'sinatra'
+  if RUBY_VERSION < '2.2.0'
+    gem 'sinatra', '~> 1.4.8'
+  else
+    gem 'sinatra'
+  end
   gem 'shotgun'
 end

data/lib/rouge.rb

@@ -34,51 +34,55 @@ module Rouge
   end
 end
 
-load_dir = Pathname.new(__FILE__).dirname
-load load_dir.join('rouge/version.rb')
+# mimic Kernel#require_relative API
+def load_relative(path)
+  load File.join(__dir__, "#{path}.rb")
+end
 
-load load_dir.join('rouge/util.rb')
+def lexer_dir(path = '')
+  File.join(__dir__, 'rouge', 'lexers', path)
+end
 
-load load_dir.join('rouge/text_analyzer.rb')
-load load_dir.join('rouge/token.rb')
+load_relative 'rouge/version'
+load_relative 'rouge/util'
+load_relative 'rouge/text_analyzer'
+load_relative 'rouge/token'
 
-load load_dir.join('rouge/lexer.rb')
-load load_dir.join('rouge/regex_lexer.rb')
-load load_dir.join('rouge/template_lexer.rb')
+load_relative 'rouge/lexer'
+load_relative 'rouge/regex_lexer'
+load_relative 'rouge/template_lexer'
 
-lexers_dir = load_dir.join('rouge/lexers')
-Dir.glob(lexers_dir.join('*.rb')).each do |f|
-  Rouge::Lexers.load_lexer(Pathname.new(f).relative_path_from(lexers_dir).to_s)
-end
+Dir.glob(lexer_dir('*rb')).each { |f| Rouge::Lexers.load_lexer(f.sub(lexer_dir, '')) }
 
-load load_dir.join('rouge/guesser.rb')
-load load_dir.join('rouge/guessers/util.rb')
-load load_dir.join('rouge/guessers/glob_mapping.rb')
-load load_dir.join('rouge/guessers/modeline.rb')
-load load_dir.join('rouge/guessers/filename.rb')
-load load_dir.join('rouge/guessers/mimetype.rb')
-load load_dir.join('rouge/guessers/source.rb')
-load load_dir.join('rouge/guessers/disambiguation.rb')
+load_relative 'rouge/guesser'
+load_relative 'rouge/guessers/util'
+load_relative 'rouge/guessers/glob_mapping'
+load_relative 'rouge/guessers/modeline'
+load_relative 'rouge/guessers/filename'
+load_relative 'rouge/guessers/mimetype'
+load_relative 'rouge/guessers/source'
+load_relative 'rouge/guessers/disambiguation'
 
-load load_dir.join('rouge/formatter.rb')
-load load_dir.join('rouge/formatters/html.rb')
-load load_dir.join('rouge/formatters/html_table.rb')
-load load_dir.join('rouge/formatters/html_pygments.rb')
-load load_dir.join('rouge/formatters/html_legacy.rb')
-load load_dir.join('rouge/formatters/html_linewise.rb')
-load load_dir.join('rouge/formatters/html_inline.rb')
-load load_dir.join('rouge/formatters/terminal256.rb')
-load load_dir.join('rouge/formatters/null.rb')
+load_relative 'rouge/formatter'
+load_relative 'rouge/formatters/html'
+load_relative 'rouge/formatters/html_table'
+load_relative 'rouge/formatters/html_pygments'
+load_relative 'rouge/formatters/html_legacy'
+load_relative 'rouge/formatters/html_linewise'
+load_relative 'rouge/formatters/html_inline'
+load_relative 'rouge/formatters/terminal256'
+load_relative 'rouge/formatters/null'
 
-load load_dir.join('rouge/theme.rb')
-load load_dir.join('rouge/themes/thankful_eyes.rb')
-load load_dir.join('rouge/themes/colorful.rb')
-load load_dir.join('rouge/themes/base16.rb')
-load load_dir.join('rouge/themes/github.rb')
-load load_dir.join('rouge/themes/igor_pro.rb')
-load load_dir.join('rouge/themes/monokai.rb')
-load load_dir.join('rouge/themes/molokai.rb')
-load load_dir.join('rouge/themes/monokai_sublime.rb')
-load load_dir.join('rouge/themes/gruvbox.rb')
-load load_dir.join('rouge/themes/tulip.rb')
-load load_dir.join('rouge/themes/pastie.rb')
+load_relative 'rouge/theme'
+load_relative 'rouge/themes/thankful_eyes'
+load_relative 'rouge/themes/colorful'
+load_relative 'rouge/themes/base16'
+load_relative 'rouge/themes/github'
+load_relative 'rouge/themes/igor_pro'
+load_relative 'rouge/themes/monokai'
+load_relative 'rouge/themes/molokai'
+load_relative 'rouge/themes/monokai_sublime'
+load_relative 'rouge/themes/gruvbox'
+load_relative 'rouge/themes/tulip'
+load_relative 'rouge/themes/pastie'
+load_relative 'rouge/themes/bw'

data/lib/rouge/cli.rb

@@ -186,6 +186,11 @@ module Rouge
         yield %[]
         yield %[--require|-r <filename>     require a filename or library before]
         yield %[                            highlighting]
+        yield %[]
+        yield %[--escape                    allow the use of escapes between <! and !>]
+        yield %[]
+        yield %[--escape-with <l> <r>       allow the use of escapes between custom]
+        yield %[                            delimiters. implies --escape]
       end
 
       def self.parse(argv)
@@ -218,6 +223,10 @@ module Rouge
             opts[:css_class] = argv.shift
           when '--lexer-opts', '-L'
             opts[:lexer_opts] = parse_cgi(argv.shift)
+          when '--escape'
+            opts[:escape] = ['<!', '!>']
+          when '--escape-with'
+            opts[:escape] = [argv.shift, argv.shift]
           when /^--/
             error! "unknown option #{arg.inspect}"
           else
@@ -244,11 +253,23 @@ module Rouge
         )
       end
 
+      def raw_lexer
+        lexer_class.new(@lexer_opts)
+      end
+
+      def escape_lexer
+        Rouge::Lexers::Escape.new(
+          start: @escape[0],
+          end: @escape[1],
+          lang: raw_lexer,
+        )
+      end
+
       def lexer
-        @lexer ||= lexer_class.new(@lexer_opts)
+        @lexer ||= @escape ? escape_lexer : raw_lexer
       end
 
-      attr_reader :input_file, :lexer_name, :mimetype, :formatter
+      attr_reader :input_file, :lexer_name, :mimetype, :formatter, :escape
 
       def initialize(opts={})
         Rouge::Lexer.enable_debug!
@@ -281,9 +302,12 @@ module Rouge
         else
           error! "unknown formatter preset #{opts[:formatter]}"
         end
+
+        @escape = opts[:escape]
       end
 
       def run
+        Formatter.enable_escape! if @escape
         formatter.format(lexer.lex(input), &method(:print))
       end
 

data/lib/rouge/demos/escape

@@ -0,0 +1,3 @@
+If Formatter.enable_escape! is called, this allows escaping into html
+or the parent format with a special delimiter. For example:
+<!<span style="text-decoration: underline">!>underlined text!<!</span>!>

data/lib/rouge/demos/supercollider

@@ -0,0 +1,11 @@
+// modulate a sine frequency and a noise amplitude with another sine
+// whose frequency depends on the horizontal mouse pointer position
+~myFunction = {
+        var x = SinOsc.ar(MouseX.kr(1, 100));
+        SinOsc.ar(300 * x + 800, 0, 0.1)
+        +
+        PinkNoise.ar(0.1 * x + 0.1)
+};
+
+~myFunction.play;
+"that's all, folks!".postln;

data/lib/rouge/demos/xojo

@@ -0,0 +1,13 @@
+Dim f As FolderItem
+f = GetOpenFolderItem(FileTypes1.jpeg) // defined in the File Type Set editor
+If not f.Exists Then
+  Beep
+  MsgBox("The file " + f.NativePath + "doesn't ""exist.""")
+Else // document exists
+  ImageWell1.image=Picture.Open(f)
+End If
+if f isa folderitem then
+  msgbox(f.name)
+end if
+Exception err As NilObjectException
+  MsgBox("Invalid pathname!")

data/lib/rouge/formatter.rb

@@ -21,6 +21,26 @@ module Rouge
       REGISTRY[tag]
     end
 
+    def self.with_escape
+      Thread.current[:'rouge/with-escape'] = true
+      yield
+    ensure
+      Thread.current[:'rouge/with-escape'] = false
+    end
+
+    def self.escape_enabled?
+      !!(@escape_enabled || Thread.current[:'rouge/with-escape'])
+    end
+
+    def self.enable_escape!
+      @escape_enabled = true
+    end
+
+    def self.disable_escape!
+      @escape_enabled = false
+      Thread.current[:'rouge/with-escape'] = false
+    end
+
     # Format a token stream.  Delegates to {#format}.
     def self.format(tokens, *a, &b)
       new(*a).format(tokens, &b)
@@ -30,8 +50,24 @@ module Rouge
       # pass
     end
 
+    def escape?(tok)
+      tok == Token::Tokens::Escape
+    end
+
+    def filter_escapes(tokens)
+      tokens.each do |t, v|
+        if t == Token::Tokens::Escape
+          yield Token::Tokens::Error, v
+        else
+          yield t, v
+        end
+      end
+    end
+
     # Format a token stream.
     def format(tokens, &b)
+      tokens = enum_for(:filter_escapes, tokens) unless Formatter.escape_enabled?
+
       return stream(tokens, &b) if block_given?
 
       out = String.new('')

data/lib/rouge/formatters/html.rb

@@ -13,6 +13,8 @@ module Rouge
       end
 
       def span(tok, val)
+        return val if escape?(tok)
+
         safe_span(tok, val.gsub(/[&<>]/, TABLE_FOR_ESCAPE_HTML))
       end
 

data/lib/rouge/formatters/html_linewise.rb

@@ -6,23 +6,18 @@ module Rouge
     class HTMLLinewise < Formatter
       def initialize(formatter, opts={})
         @formatter = formatter
+        @tag_name = opts.fetch(:tag_name, 'div')
         @class_format = opts.fetch(:class, 'line-%i')
       end
 
       def stream(tokens, &b)
-        token_lines(tokens) do |line|
-          yield "<div class=#{next_line_class}>"
-          line.each do |tok, val|
-            yield @formatter.span(tok, val)
-          end
-          yield '</div>'
+        lineno = 0
+        token_lines(tokens) do |line_tokens|
+          yield %(<#{@tag_name} class="#{sprintf @class_format, lineno += 1}">)
+          @formatter.stream(line_tokens) {|formatted| yield formatted }
+          yield %(\n</#{@tag_name}>)
         end
       end
-
-      def next_line_class
-        @lineno ||= 0
-        sprintf(@class_format, @lineno += 1).inspect
-      end
     end
   end
 end

data/lib/rouge/formatters/html_table.rb

@@ -16,46 +16,35 @@ module Rouge
       end
 
       def style(scope)
-        yield "#{scope} .rouge-table { border-spacing: 0 }"
-        yield "#{scope} .rouge-gutter { text-align: right }"
+        yield %(#{scope} .rouge-table { border-spacing: 0 })
+        yield %(#{scope} .rouge-gutter { text-align: right })
       end
 
       def stream(tokens, &b)
-        num_lines = 0
-        last_val = ''
-        formatted = String.new('')
-
-        tokens.each do |tok, val|
-          last_val = val
-          num_lines += val.scan(/\n/).size
-          formatted << @inner.span(tok, val)
-        end
-
-        # add an extra line for non-newline-terminated strings
-        if last_val[-1] != "\n"
+        last_val = nil
+        num_lines = tokens.reduce(0) {|count, (_, val)| count + (last_val = val).count(?\n) }
+        formatted = @inner.format(tokens)
+        unless last_val && last_val.end_with?(?\n)
           num_lines += 1
-          @inner.span(Token::Tokens::Text::Whitespace, "\n") { |str| formatted << str }
+          formatted << ?\n
         end
 
         # generate a string of newline-separated line numbers for the gutter>
-        formatted_line_numbers = (@start_line..num_lines+@start_line-1).map do |i|
-          sprintf("#{@line_format}", i) << "\n"
-        end.join('')
-
-        numbers = %(<pre class="lineno">#{formatted_line_numbers}</pre>)
-
-        yield %(<table class="#@table_class"><tbody><tr>)
+        formatted_line_numbers = (@start_line..(@start_line + num_lines - 1)).map do |i|
+          sprintf(@line_format, i)
+        end.join(?\n) << ?\n
 
+        buffer = [%(<table class="#@table_class"><tbody><tr>)]
         # the "gl" class applies the style for Generic.Lineno
-        yield %(<td class="#@gutter_class gl">)
-        yield numbers
-        yield '</td>'
-
-        yield %(<td class="#@code_class"><pre>)
-        yield formatted
-        yield '</pre></td>'
-
-        yield "</tr></tbody></table>"
+        buffer << %(<td class="#@gutter_class gl">)
+        buffer << %(<pre class="lineno">#{formatted_line_numbers}</pre>)
+        buffer << '</td>'
+        buffer << %(<td class="#@code_class"><pre>)
+        buffer << formatted
+        buffer << '</pre></td>'
+        buffer << '</tr></tbody></table>'
+
+        yield buffer.join
       end
     end
   end

data/lib/rouge/formatters/terminal256.rb

@@ -159,6 +159,7 @@ module Rouge
 
     # private
       def escape_sequence(token)
+        return '' if escape?(token)
         @escape_sequences ||= {}
         @escape_sequences[token.qualname] ||=
           EscapeSequence.new(get_style(token))

data/lib/rouge/guessers/disambiguation.rb

@@ -101,6 +101,19 @@ module Rouge
 
         Cpp
       end
+
+      disambiguate '*.plist' do
+        next XML if matches?(/\A<\?xml\b/)
+
+        Plist
+      end
+
+      disambiguate '*.sc' do
+        next Python if matches?(/^#/)
+        next SuperCollider if matches?(/(?:^~|;$)/)
+
+        next Python
+      end
     end
   end
 end

data/lib/rouge/guessers/source.rb

@@ -22,7 +22,7 @@ module Rouge
         source_text = TextAnalyzer.new(source_text)
 
         collect_best(lexers) do |lexer|
-          next unless lexer.methods(false).include? :detect?
+          next unless lexer.detectable?
           lexer.detect?(source_text) ? 1 : nil
         end
       end

data/lib/rouge/lexer.rb

@@ -23,6 +23,14 @@ module Rouge
         new(opts).lex(stream, &b)
       end
 
+      # In case #continue_lex is called statically, we simply
+      # begin a new lex from the beginning, since there is no state.
+      #
+      # @see #continue_lex
+      def continue_lex(*a, &b)
+        lex(*a, &b)
+      end
+
       # Given a name in string, return the correct lexer class.
       # @param [String] name
       # @return [Class<Rouge::Lexer>,nil]
@@ -118,7 +126,7 @@ module Rouge
 
       # @return a list of all lexers.
       def all
-        registry.values.uniq
+        @all ||= registry.values.uniq
       end
 
       # Guess which lexer to use based on a hash of info.
@@ -188,16 +196,24 @@ module Rouge
       end
 
       def disable_debug!
-        @debug_enabled = false
+        remove_instance_variable :@debug_enabled
       end
 
       def debug_enabled?
-        !!@debug_enabled
+        (defined? @debug_enabled) ? true : false
+      end
+
+      # Determine if a lexer has a method named +:detect?+ defined in its
+      # singleton class.
+      def detectable?
+        @detectable ||= methods(false).include?(:detect?)
       end
 
     protected
       # @private
       def register(name, lexer)
+        # reset an existing list of lexers
+        @all = nil if @all
         registry[name.to_s] = lexer
       end
 
@@ -236,6 +252,13 @@ module Rouge
 
       # Specify a list of filename globs associated with this lexer.
       #
+      # If a filename glob is associated with more than one lexer, this can
+      # cause a Guesser::Ambiguous error to be raised in various guessing
+      # methods. These errors can be avoided by disambiguation. Filename globs
+      # are disambiguated in one of two ways. Either the lexer will define a
+      # `self.detect?` method (intended for use with shebangs and doctypes) or a
+      # manual rule will be specified in Guessers::Disambiguation.
+      #
       # @example
       #   class Ruby < Lexer
       #     filenames '*.rb', '*.ruby', 'Gemfile', 'Rakefile'
@@ -256,7 +279,9 @@ module Rouge
 
       # @private
       def assert_utf8!(str)
-        return if %w(US-ASCII UTF-8 ASCII-8BIT).include? str.encoding.name
+        encoding = str.encoding.name
+        return if encoding == 'US-ASCII' || encoding == 'UTF-8' || encoding == 'ASCII-8BIT'
+
         raise EncodingError.new(
           "Bad encoding: #{str.encoding.names.join(',')}. " +
           "Please convert your string to UTF-8."
@@ -390,15 +415,23 @@ module Rouge
 
     # Given a string, yield [token, chunk] pairs.  If no block is given,
     # an enumerator is returned.
-    #
-    # @option opts :continue
-    #   Continue the lex from the previous state (i.e. don't call #reset!)
-    def lex(string, opts={}, &b)
-      return enum_for(:lex, string, opts) unless block_given?
+    def lex(string, opts=nil, &b)
+      if opts
+        warn 'the :continue option to Formatter#lex is deprecated, use #continue_lex instead.'
+        return continue_lex(string, &b)
+      end
+
+      return enum_for(:lex, string) unless block_given?
 
       Lexer.assert_utf8!(string)
+      reset!
 
-      reset! unless opts[:continue]
+      continue_lex(string, &b)
+    end
+
+    # Continue the lex from the the current state without resetting
+    def continue_lex(string, &b)
+      return enum_for(:continue_lex, string, &b) unless block_given?
 
       # consolidate consecutive tokens of the same type
       last_token = nil
@@ -454,9 +487,7 @@ module Rouge
     def self.load_lexer(relpath)
       return if @_loaded_lexers.key?(relpath)
       @_loaded_lexers[relpath] = true
-
-      root = Pathname.new(__FILE__).dirname.join('lexers')
-      load root.join(relpath)
+      load File.join(__dir__, 'lexers', relpath)
     end
   end
 end