rouge 2.2.1 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 35a33896b2a596c94fc465bae5ef1e65874fd6b3
-  data.tar.gz: 5b0c4180e0fc5748e8a35d139d3f37a8e0030d7e
+  metadata.gz: 105d563933ea57ff19cca03ab1fd98fc3b01ea3a
+  data.tar.gz: 72e6098b1419ff753ddb8bf748d6822d5fc5c1a5
 SHA512:
-  metadata.gz: 9d461f6335a3b1df5312cc0c259d49a9a5a0e21f0752028898eebbb736913f39bf04a28a91578579ef84362280f1945eef8464a682c7c141a969817172b48de7
-  data.tar.gz: 1f0dbb4f04334ea77858b6bcf3c3795abf82597cac61ef2a08ff08f526fe6fa47ba64a600715ec94c52bb59157ad9b48c73ad5fae600aed80a541c744270dd65
+  metadata.gz: 6346ab18ab19b7758e3f37eec75ddeac72a11edacd4416783dbfaccb33e3aadda9841373db11436a9fe4fbdd457a8bb2f4575f79b2c546e596b9d9cc829612e3
+  data.tar.gz: 9df1ed34e20dfbb2733ebef31d80792547913afdff931c556e73ff731875968233f12e1c62e91cfbaa84d4653b684aeec5e9a4778ff16fb3c5279585b974a846

data/Gemfile

@@ -3,12 +3,12 @@ source 'http://rubygems.org'
 gemspec
 
 gem 'bundler', '~> 1.15'
-gem 'rake', '~> 12.0'
+gem 'rake'
 
-gem 'minitest', '~> 4.0'
-gem 'wrong'
+gem 'minitest', '>= 5.0'
+gem 'minitest-power_assert'
 
-gem 'rubocop', '~> 0.49.1' if RUBY_VERSION >= '2.0.0'
+gem 'rubocop', '~> 0.49.1'
 
 # don't try to install redcarpet under jruby
 gem 'redcarpet', :platforms => :ruby

data/lib/rouge.rb

@@ -41,13 +41,6 @@ load load_dir.join('rouge/util.rb')
 load load_dir.join('rouge/text_analyzer.rb')
 load load_dir.join('rouge/token.rb')
 
-load load_dir.join('rouge/guesser.rb')
-load load_dir.join('rouge/guessers/glob_mapping.rb')
-load load_dir.join('rouge/guessers/modeline.rb')
-load load_dir.join('rouge/guessers/filename.rb')
-load load_dir.join('rouge/guessers/mimetype.rb')
-load load_dir.join('rouge/guessers/source.rb')
-
 load load_dir.join('rouge/lexer.rb')
 load load_dir.join('rouge/regex_lexer.rb')
 load load_dir.join('rouge/template_lexer.rb')
@@ -57,6 +50,15 @@ Dir.glob(lexers_dir.join('*.rb')).each do |f|
   Rouge::Lexers.load_lexer(Pathname.new(f).relative_path_from(lexers_dir).to_s)
 end
 
+load load_dir.join('rouge/guesser.rb')
+load load_dir.join('rouge/guessers/util.rb')
+load load_dir.join('rouge/guessers/glob_mapping.rb')
+load load_dir.join('rouge/guessers/modeline.rb')
+load load_dir.join('rouge/guessers/filename.rb')
+load load_dir.join('rouge/guessers/mimetype.rb')
+load load_dir.join('rouge/guessers/source.rb')
+load load_dir.join('rouge/guessers/disambiguation.rb')
+
 load load_dir.join('rouge/formatter.rb')
 load load_dir.join('rouge/formatters/html.rb')
 load load_dir.join('rouge/formatters/html_table.rb')

data/lib/rouge/cli.rb

@@ -13,9 +13,9 @@ module Rouge
     def file
       case input
       when '-'
-        IO.new($stdin.fileno, 'r:utf-8')
+        IO.new($stdin.fileno, 'rt:bom|utf-8')
       when String
-        File.new(input, 'r:utf-8')
+        File.new(input, 'rt:bom|utf-8')
       when ->(i){ i.respond_to? :read }
         input
       end
@@ -44,6 +44,7 @@ module Rouge
       yield %|	help		#{Help.desc}|
       yield %|	style		#{Style.desc}|
       yield %|	list		#{List.desc}|
+      yield %|	guess		#{Guess.desc}|
       yield %|	version		#{Version.desc}|
       yield %||
       yield %|See `rougify help <command>` for more info.|
@@ -97,6 +98,8 @@ module Rouge
         Style
       when 'list'
         List
+      when 'guess'
+        Guess
       end
     end
 
@@ -375,6 +378,37 @@ module Rouge
       end
     end
 
+    class Guess < CLI
+      def self.desc
+        "guess the languages of file"
+      end
+
+      def self.parse(args)
+        new(input_file: args.shift)
+      end
+
+      attr_reader :input_file, :input_source
+
+      def initialize(opts)
+        @input_file = opts[:input_file] || '-'
+        @input_source = FileReader.new(@input_file).read
+      end
+
+      def lexers
+        Lexer.guesses(
+          filename: input_file,
+          source: input_source,
+        )
+      end
+
+      def run
+        lexers.each do |l|
+          puts "{ tag: #{l.tag.inspect}, title: #{l.title.inspect}, desc: #{l.desc.inspect} }"
+        end
+      end
+    end
+
+
   private_class_method
     def self.normalize_syntax(argv)
       out = []

data/lib/rouge/guessers/disambiguation.rb

@@ -0,0 +1,88 @@
+module Rouge
+  module Guessers
+    class Disambiguation < Guesser
+      include Util
+      include Lexers
+
+      def initialize(filename, source)
+        @filename = File.basename(filename)
+        @source = source
+      end
+
+      def filter(lexers)
+        return lexers if lexers.size == 1
+        return lexers if lexers.size == Lexer.all.size
+
+        @analyzer = TextAnalyzer.new(get_source(@source))
+
+        self.class.disambiguators.each do |disambiguator|
+          next unless disambiguator.match?(@filename)
+
+          filtered = disambiguator.decide!(self)
+          return filtered if filtered
+        end
+
+        return lexers
+      end
+
+      def contains?(text)
+        return @analyzer.include?(text)
+      end
+
+      def matches?(re)
+        return !!(@analyzer =~ re)
+      end
+
+      @disambiguators = []
+      def self.disambiguate(*patterns, &decider)
+        @disambiguators << Disambiguator.new(patterns, &decider)
+      end
+
+      def self.disambiguators
+        @disambiguators
+      end
+
+      class Disambiguator
+        include Util
+
+        def initialize(patterns, &decider)
+          @patterns = patterns
+          @decider = decider
+        end
+
+        def decide!(guesser)
+          out = guesser.instance_eval(&@decider)
+          case out
+          when Array then out
+          when nil then nil
+          else [out]
+          end
+        end
+
+        def match?(filename)
+          @patterns.any? { |p| test_glob(p, filename) }
+        end
+      end
+
+      disambiguate '*.pl' do
+        next Perl if contains?('my $')
+        next Prolog if contains?(':-')
+        next Prolog if matches?(/\A\w+(\(\w+\,\s*\w+\))*\./)
+      end
+
+      disambiguate '*.h' do
+        next ObjectiveC if matches?(/@(end|implementation|protocol|property)\b/)
+        next ObjectiveC if contains?('@"')
+
+        C
+      end
+
+      disambiguate '*.m' do
+        next ObjectiveC if matches?(/@(end|implementation|protocol|property)\b/)
+        next ObjectiveC if contains?('@"')
+
+        next Matlab if matches?(/^\s*?%/)
+      end
+    end
+  end
+end

data/lib/rouge/guessers/glob_mapping.rb

@@ -3,6 +3,8 @@ module Rouge
     # This class allows for custom behavior
     # with glob -> lexer name mappings
     class GlobMapping < Guesser
+      include Util
+
       def self.by_pairs(mapping, filename)
         glob_map = {}
         mapping.each do |(glob, lexer_name)|
@@ -29,18 +31,13 @@ module Rouge
 
         collect_best(lexers) do |lexer|
           score = (@glob_map[lexer.name] || []).map do |pattern|
-            if test_pattern(pattern, basename)
+            if test_glob(pattern, basename)
               # specificity is better the fewer wildcards there are
               -pattern.scan(/[*?\[]/).size
             end
           end.compact.min
         end
       end
-
-      private
-      def test_pattern(pattern, path)
-        File.fnmatch?(pattern, path, File::FNM_DOTMATCH | File::FNM_CASEFOLD)
-      end
     end
   end
 end

data/lib/rouge/guessers/modeline.rb

@@ -1,6 +1,8 @@
 module Rouge
   module Guessers
     class Modeline < Guesser
+      include Util
+
       # [jneen] regexen stolen from linguist
       EMACS_MODELINE = /-\*-\s*(?:(?!mode)[\w-]+\s*:\s*(?:[\w+-]+)\s*;?\s*)*(?:mode\s*:)?\s*([\w+-]+)\s*(?:;\s*(?!mode)[\w-]+\s*:\s*[\w+-]+\s*)*;?\s*-\*-/i
 
@@ -25,10 +27,9 @@ module Rouge
         # don't bother reading the stream if we've already decided
         return lexers if lexers.size == 1
 
-        source_text = @source
-        source_text = source_text.read if source_text.respond_to? :read
+        source_text = get_source(@source)
 
-        lines = source_text.split(/\r?\n/)
+        lines = source_text.split(/\n/)
 
         search_space = (lines.first(@lines) + lines.last(@lines)).join("\n")
 

data/lib/rouge/guessers/source.rb

@@ -1,6 +1,8 @@
 module Rouge
   module Guessers
     class Source < Guesser
+      include Util
+
       attr_reader :source
       def initialize(source)
         @source = source
@@ -11,27 +13,15 @@ module Rouge
         # we've already filtered to 1
         return lexers if lexers.size == 1
 
-        # If we're filtering against *all* lexers, we only use confident return
-        # values from analyze_text.  But if we've filtered down already, we can trust
-        # the analysis more.
-        threshold = lexers.size < 10 ? 0 : 0.5
-
-        source_text = case @source
-        when String
-          @source
-        when ->(s){ s.respond_to? :read }
-          @source.read
-        else
-          raise 'invalid source'
-        end
+        source_text = get_source(@source)
 
         Lexer.assert_utf8!(source_text)
 
         source_text = TextAnalyzer.new(source_text)
 
-        collect_best(lexers, threshold: threshold) do |lexer|
-          next unless lexer.methods(false).include? :analyze_text
-          lexer.analyze_text(source_text)
+        collect_best(lexers) do |lexer|
+          next unless lexer.methods(false).include? :detect?
+          lexer.detect?(source_text) ? 1 : nil
         end
       end
     end

data/lib/rouge/guessers/util.rb

@@ -0,0 +1,32 @@
+module Rouge
+  module Guessers
+    module Util
+      module SourceNormalizer
+        UTF8_BOM = "\xEF\xBB\xBF"
+        UTF8_BOM_RE = /\A#{UTF8_BOM}/
+
+        # @param [String,nil] source
+        # @return [String,nil]
+        def self.normalize(source)
+          source.sub(UTF8_BOM_RE, '').gsub(/\r\n/, "\n")
+        end
+      end
+
+      def test_glob(pattern, path)
+        File.fnmatch?(pattern, path, File::FNM_DOTMATCH | File::FNM_CASEFOLD)
+      end
+
+      # @param [String,IO] source
+      # @return [String]
+      def get_source(source)
+        if source.respond_to?(:to_str)
+          SourceNormalizer.normalize(source.to_str)
+        elsif source.respond_to?(:read)
+          SourceNormalizer.normalize(source.read)
+        else
+          raise ArgumentError, "Invalid source: #{source.inspect}"
+        end
+      end
+    end
+  end
+end

data/lib/rouge/lexer.rb

@@ -22,7 +22,9 @@ module Rouge
         new(opts).lex(stream, &b)
       end
 
-      # Given a string, return the correct lexer class.
+      # Given a name in string, return the correct lexer class.
+      # @param [String] name
+      # @return [Class<Rouge::Lexer>,nil]
       def find(name)
         registry[name.to_s]
       end
@@ -42,6 +44,7 @@ module Rouge
       # markdown lexer for highlighting internal code blocks.
       #
       def find_fancy(str, code=nil, additional_options={})
+
         if str && !str.include?('?') && str != 'guess'
           lexer_class = find(str)
           return lexer_class && lexer_class.new(additional_options)
@@ -109,7 +112,7 @@ module Rouge
       def demo(arg=:absent)
         return @demo = arg unless arg == :absent
 
-        @demo = File.read(demo_file, encoding: 'utf-8')
+        @demo = File.read(demo_file, mode: 'rt:bom|utf-8')
       end
 
       # @return a list of all lexers.
@@ -133,6 +136,7 @@ module Rouge
         guessers << Guessers::Filename.new(filename) if filename
         guessers << Guessers::Modeline.new(source) if source
         guessers << Guessers::Source.new(source) if source
+        guessers << Guessers::Disambiguation.new(filename, source) if source && filename
 
         Guesser.guess(guessers, Lexer.all)
       end
@@ -147,16 +151,23 @@ module Rouge
       #   The source itself, which, if guessing by mimetype or filename
       #   fails, will be searched for shebangs, <!DOCTYPE ...> tags, and
       #   other hints.
+      # @param [Proc] fallback called if multiple lexers are detected.
+      #   If omitted, Guesser::Ambiguous is raised.
       #
-      # @see Lexer.analyze_text
+      # @see Lexer.detect?
       # @see Lexer.guesses
-      def guess(info={})
+      # @return [Class<Rouge::Lexer>]
+      def guess(info={}, &fallback)
         lexers = guesses(info)
 
         return Lexers::PlainText if lexers.empty?
         return lexers[0] if lexers.size == 1
 
-        raise Guesser::Ambiguous.new(lexers)
+        if fallback
+          fallback.call(lexers)
+        else
+          raise Guesser::Ambiguous.new(lexers)
+        end
       end
 
       def guess_by_mimetype(mt)
@@ -425,16 +436,14 @@ module Rouge
 
     # @abstract
     #
-    # Return a number between 0 and 1 indicating the likelihood that
-    # the text given should be lexed with this lexer.  The default
-    # implementation returns 0.  Values under 0.5 will only be used
-    # to disambiguate filename or mimetype matches.
+    # Return true if there is an in-text indication (such as a shebang
+    # or DOCTYPE declaration) that this lexer should be used.
     #
     # @param [TextAnalyzer] text
     #   the text to be analyzed, with a couple of handy methods on it,
     #   like {TextAnalyzer#shebang?} and {TextAnalyzer#doctype?}
-    def self.analyze_text(text)
-      0
+    def self.detect?(text)
+      false
     end
   end
 

data/lib/rouge/lexers/apiblueprint.rb

@@ -11,10 +11,6 @@ module Rouge
       filenames '*.apib'
       mimetypes 'text/vnd.apiblueprint'
 
-      def self.analyze_text(text)
-        return 1 if text.start_with?('FORMAT: 1A\n')
-      end
-
       prepend :root do
         # Metadata
         rule(/(\S+)(:\s*)(.*)$/) do

data/lib/rouge/lexers/awk.rb

@@ -10,8 +10,8 @@ module Rouge
       filenames '*.awk'
       mimetypes 'application/x-awk'
 
-      def self.analyze_text(text)
-        return 1 if text.shebang?('awk')
+      def self.detect?(text)
+        return true if text.shebang?('awk')
       end
 
       id = /[$a-zA-Z_][a-zA-Z0-9_]*/

data/lib/rouge/lexers/biml.rb

@@ -8,8 +8,8 @@ module Rouge
       tag 'biml'
       filenames '*.biml'
 
-      def self.analyze_text(text)
-        return 1 if text =~ /<\s*Biml\b/
+      def self.detect?(text)
+        return true if text =~ /<\s*Biml\b/
       end
 
       prepend :root do