rouge 1.11.0 → 1.11.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0d52689800e9b41571b3c09b3412f672a0e96de8
-  data.tar.gz: f0b4629db72f7122e8ff0760487f9193edec8f9c
+  metadata.gz: 0efe4d1a5c6a98c1b4a532ef1b703ee60ceb6a1c
+  data.tar.gz: 6fa2c0ac804354f3e058137869ab151b922c0864
 SHA512:
-  metadata.gz: cf7d9859a295ad5e9e269eecf77e3477d3bf93b062f19e0095b81135e66265375ddbfb1c986a155a4ed84921366ce2914e0ee900040c24f58d488921417a8928
-  data.tar.gz: b4de6ef45285cf75dac6d9558341955e5c29b30f06ea6a7584c99b1febde9a63915fa2a5442839fa3c7cc51da57dd51a2152adcfb98f09e83abba0524e11c359
+  metadata.gz: ab710b89b387ac5675cd31467023eff9600fd7131b51966fe72394569e1ec9f92f3bb70b8448a51e11de12bd83b9217b2b364d120ec5faa4e2a46626371d591d
+  data.tar.gz: 902c5c1d3c88a160b8e140b6f8140a53627e10334ba3e4b628c44da1c057406cc374e3e21f8cbb6b74cba04bcd5e58c54cc6714fa8fe83484ae2d0d47f73fbcd

data/lib/rouge.rb

@@ -36,6 +36,13 @@ load load_dir.join('rouge/util.rb')
 load load_dir.join('rouge/text_analyzer.rb')
 load load_dir.join('rouge/token.rb')
 
+load load_dir.join('rouge/guesser.rb')
+load load_dir.join('rouge/guessers/glob_mapping.rb')
+load load_dir.join('rouge/guessers/modeline.rb')
+load load_dir.join('rouge/guessers/filename.rb')
+load load_dir.join('rouge/guessers/mimetype.rb')
+load load_dir.join('rouge/guessers/source.rb')
+
 load load_dir.join('rouge/lexer.rb')
 load load_dir.join('rouge/regex_lexer.rb')
 load load_dir.join('rouge/template_lexer.rb')
@@ -58,3 +65,4 @@ load load_dir.join('rouge/themes/github.rb')
 load load_dir.join('rouge/themes/monokai.rb')
 load load_dir.join('rouge/themes/molokai.rb')
 load load_dir.join('rouge/themes/monokai_sublime.rb')
+load load_dir.join('rouge/themes/gruvbox.rb')

data/lib/rouge/cli.rb

@@ -13,9 +13,9 @@ module Rouge
     def file
       case input
       when '-'
-        $stdin
+        IO.new($stdin.fileno, 'r:utf-8')
       when String
-        File.new(input)
+        File.new(input, 'r:utf-8')
       when ->(i){ i.respond_to? :read }
         input
       end
@@ -23,7 +23,7 @@ module Rouge
 
     def read
       @read ||= begin
-        File.read(file, encoding: 'utf-8') 
+        file.read
       rescue => e
         $stderr.puts "unable to open #{input}: #{e.message}"
         exit 1

data/lib/rouge/demos/cfscript

@@ -0,0 +1,18 @@
+component accessors="true" {
+
+  property type="string" name="firstName" default="";
+  property string username;
+
+  function init(){
+    return this;
+  }
+
+  public any function submitOrder( required product, coupon="", boolean results=true ){
+
+    var foo = function( required string baz, x=true, y=false ){
+      return "bar!";
+    };
+
+    return foo;
+  }
+}

data/lib/rouge/demos/kotlin

@@ -0,0 +1,3 @@
+fun main(args: Array<String>) {
+    println("Hello, world!")
+}

data/lib/rouge/demos/pascal

@@ -0,0 +1,14 @@
+program FizzBuzz(output);
+var
+  i: Integer;
+begin
+  for i := 1 to 100 do
+    if i mod 15 = 0 then
+      WriteLn('FizzBuzz')
+    else if i mod 3 = 0 then
+      WriteLn('Fizz')
+    else if i mod 5 = 0 then
+      WriteLn('Buzz')
+    else
+      WriteLn(i)
+end.

data/lib/rouge/guesser.rb

@@ -0,0 +1,46 @@
+module Rouge
+  class Guesser
+    def self.guess(guessers, lexers)
+      original_size = lexers.size
+
+      guessers.each do |g|
+        new_lexers = case g
+        when Guesser then g.filter(lexers)
+        when proc { |x| x.respond_to? :call } then g.call(lexers)
+        else raise "bad guesser: #{g}"
+        end
+
+        lexers = new_lexers && new_lexers.any? ? new_lexers : lexers
+      end
+
+      # if we haven't filtered the input at *all*,
+      # then we have no idea what language it is,
+      # so we bail and return [].
+      lexers.size < original_size ? lexers : []
+    end
+
+    def collect_best(lexers, opts={}, &scorer)
+      best = []
+      best_score = opts[:threshold]
+
+      lexers.each do |lexer|
+        score = scorer.call(lexer)
+
+        next if score.nil?
+
+        if best_score.nil? || score > best_score
+          best_score = score
+          best = [lexer]
+        elsif score == best_score
+          best << lexer
+        end
+      end
+
+      best
+    end
+
+    def filter(lexers)
+      raise 'abstract'
+    end
+  end
+end

data/lib/rouge/guessers/filename.rb

@@ -0,0 +1,25 @@
+module Rouge
+  module Guessers
+    class Filename < Guesser
+      attr_reader :fname
+      def initialize(filename)
+        @filename = filename
+      end
+
+      # returns a list of lexers that match the given filename with
+      # equal specificity (i.e. number of wildcards in the pattern).
+      # This helps disambiguate between, e.g. the Nginx lexer, which
+      # matches `nginx.conf`, and the Conf lexer, which matches `*.conf`.
+      # In this case, nginx will win because the pattern has no wildcards,
+      # while `*.conf` has one.
+      def filter(lexers)
+        mapping = {}
+        lexers.each do |lexer|
+          mapping[lexer.name] = lexer.filenames || []
+        end
+
+        GlobMapping.new(mapping, @filename).filter(lexers)
+      end
+    end
+  end
+end

data/lib/rouge/guessers/glob_mapping.rb

@@ -0,0 +1,46 @@
+module Rouge
+  module Guessers
+    # This class allows for custom behavior
+    # with glob -> lexer name mappings
+    class GlobMapping < Guesser
+      def self.by_pairs(mapping, filename)
+        glob_map = {}
+        mapping.each do |(glob, lexer_name)|
+          lexer = Lexer.find(lexer_name)
+
+          # ignore unknown lexers
+          next unless lexer
+
+          glob_map[lexer.name] ||= []
+          glob_map[lexer.name] << glob
+        end
+
+        new(glob_map, filename)
+      end
+
+      attr_reader :glob_map, :filename
+      def initialize(glob_map, filename)
+        @glob_map = glob_map
+        @filename = filename
+      end
+
+      def filter(lexers)
+        basename = File.basename(filename)
+
+        collect_best(lexers) do |lexer|
+          score = (@glob_map[lexer.name] || []).map do |pattern|
+            if test_pattern(pattern, basename)
+              # specificity is better the fewer wildcards there are
+              -pattern.scan(/[*?\[]/).size
+            end
+          end.compact.min
+        end
+      end
+
+      private
+      def test_pattern(pattern, path)
+        File.fnmatch?(pattern, path, File::FNM_DOTMATCH | File::FNM_CASEFOLD)
+      end
+    end
+  end
+end

data/lib/rouge/guessers/mimetype.rb

@@ -0,0 +1,14 @@
+module Rouge
+  module Guessers
+    class Mimetype < Guesser
+      attr_reader :mimetype
+      def initialize(mimetype)
+        @mimetype = mimetype
+      end
+
+      def filter(lexers)
+        lexers.select { |lexer| lexer.mimetypes.include? @mimetype }
+      end
+    end
+  end
+end

data/lib/rouge/guessers/modeline.rb

@@ -0,0 +1,42 @@
+module Rouge
+  module Guessers
+    class Modeline < Guesser
+      # [jneen] regexen stolen from linguist
+      EMACS_MODELINE = /-\*-\s*(?:(?!mode)[\w-]+\s*:\s*(?:[\w+-]+)\s*;?\s*)*(?:mode\s*:)?\s*([\w+-]+)\s*(?:;\s*(?!mode)[\w-]+\s*:\s*[\w+-]+\s*)*;?\s*-\*-/i
+
+      # First form vim modeline
+      # [text]{white}{vi:|vim:|ex:}[white]{options}
+      # ex: 'vim: syntax=ruby'
+      VIM_MODELINE_1 = /(?:vim|vi|ex):\s*(?:ft|filetype|syntax)=(\w+)\s?/i
+
+      # Second form vim modeline (compatible with some versions of Vi)
+      # [text]{white}{vi:|vim:|Vim:|ex:}[white]se[t] {options}:[text]
+      # ex: 'vim set syntax=ruby:'
+      VIM_MODELINE_2 = /(?:vim|vi|Vim|ex):\s*se(?:t)?.*\s(?:ft|filetype|syntax)=(\w+)\s?.*:/i
+
+      MODELINES = [EMACS_MODELINE, VIM_MODELINE_1, VIM_MODELINE_2]
+
+      def initialize(source, opts={})
+        @source = source
+        @lines = opts[:lines] || 5
+      end
+
+      def filter(lexers)
+        # don't bother reading the stream if we've already decided
+        return lexers if lexers.size == 1
+
+        source_text = @source
+        source_text = source_text.read if source_text.respond_to? :read
+
+        lines = source_text.split(/\r?\n/)
+
+        search_space = (lines.first(@lines) + lines.last(@lines)).join("\n")
+
+        matches = MODELINES.map { |re| re.match(search_space) }.compact
+        match_set = Set.new(matches.map { |m| m[1] })
+
+        lexers.select { |l| (Set.new([l.tag] + l.aliases) & match_set).any? }
+      end
+    end
+  end
+end

data/lib/rouge/guessers/source.rb

@@ -0,0 +1,39 @@
+module Rouge
+  module Guessers
+    class Source < Guesser
+      attr_reader :source
+      def initialize(source)
+        @source = source
+      end
+
+      def filter(lexers)
+        # don't bother reading the input if
+        # we've already filtered to 1
+        return lexers if lexers.size == 1
+
+        # If we're filtering against *all* lexers, we only use confident return
+        # values from analyze_text.  But if we've filtered down already, we can trust
+        # the analysis more.
+        threshold = lexers.size < 10 ? 0 : 0.5
+
+        source_text = case @source
+        when String
+          @source
+        when ->(s){ s.respond_to? :read }
+          @source.read
+        else
+          raise 'invalid source'
+        end
+
+        Lexer.assert_utf8!(source_text)
+
+        source_text = TextAnalyzer.new(source_text)
+
+        collect_best(lexers, threshold: threshold) do |lexer|
+          next unless lexer.methods(false).include? :analyze_text
+          lexer.analyze_text(source_text)
+        end
+      end
+    end
+  end
+end

data/lib/rouge/lexer.rb

@@ -109,26 +109,17 @@ module Rouge
       # to use.
       def guesses(info={})
         mimetype, filename, source = info.values_at(:mimetype, :filename, :source)
-        lexers = registry.values.uniq
-        total_size = lexers.size
-
-        lexers = filter_by_mimetype(lexers, mimetype) if mimetype
-        return lexers if lexers.size == 1
-
-        lexers = filter_by_filename(lexers, filename) if filename
-        return lexers if lexers.size == 1
-
-        if source
-          # If we're filtering against *all* lexers, we only use confident return
-          # values from analyze_text.  But if we've filtered down already, we can trust
-          # the analysis more.
-          source_threshold = lexers.size < total_size ? 0 : 0.5
-          return [best_by_source(lexers, source, source_threshold)].compact
-        elsif lexers.size < total_size
-          return lexers
-        else
-          return []
-        end
+        custom_globs = info[:custom_globs]
+
+        guessers = (info[:guessers] || []).dup
+
+        guessers << Guessers::Mimetype.new(mimetype) if mimetype
+        guessers << Guessers::GlobMapping.by_pairs(custom_globs, filename) if custom_globs && filename
+        guessers << Guessers::Filename.new(filename) if filename
+        guessers << Guessers::Modeline.new(source) if source
+        guessers << Guessers::Source.new(source) if source
+
+        Guesser.guess(guessers, Lexer.all)
       end
 
       class AmbiguousGuess < StandardError
@@ -175,71 +166,6 @@ module Rouge
       end
 
     private
-      def filter_by_mimetype(lexers, mt)
-        filtered = lexers.select { |lexer| lexer.mimetypes.include? mt }
-        filtered.any? ? filtered : lexers
-      end
-
-      # returns a list of lexers that match the given filename with
-      # equal specificity (i.e. number of wildcards in the pattern).
-      # This helps disambiguate between, e.g. the Nginx lexer, which
-      # matches `nginx.conf`, and the Conf lexer, which matches `*.conf`.
-      # In this case, nginx will win because the pattern has no wildcards,
-      # while `*.conf` has one.
-      def filter_by_filename(lexers, fname)
-        fname = File.basename(fname)
-
-        out = []
-        best_seen = nil
-        lexers.each do |lexer|
-          score = lexer.filenames.map do |pattern|
-            if File.fnmatch?(pattern, fname, File::FNM_DOTMATCH)
-              # specificity is better the fewer wildcards there are
-              pattern.scan(/[*?\[]/).size
-            end
-          end.compact.min
-
-          next unless score
-
-          if best_seen.nil? || score < best_seen
-            best_seen = score
-            out = [lexer]
-          elsif score == best_seen
-            out << lexer
-          end
-        end
-
-        out.any? ? out : lexers
-      end
-
-      def best_by_source(lexers, source, threshold=0)
-        source = case source
-        when String
-          source
-        when ->(s){ s.respond_to? :read }
-          source.read
-        else
-          raise 'invalid source'
-        end
-
-        assert_utf8!(source)
-
-        source = TextAnalyzer.new(source)
-
-        best_result = threshold
-        best_match = nil
-        lexers.each do |lexer|
-          result = lexer.analyze_text(source) || 0
-          return lexer if result == 1
-
-          if result > best_result
-            best_match = lexer
-            best_result = result
-          end
-        end
-
-        best_match
-      end
 
     protected
       # @private

data/lib/rouge/lexers/cfscript.rb

@@ -0,0 +1,153 @@
+# -*- coding: utf-8 -*- #
+
+module Rouge
+  module Lexers
+
+    class Cfscript < RegexLexer
+      title "CFScript"
+      desc 'CFScript, the CFML scripting language'
+      tag 'cfscript'
+      aliases 'cfc'
+      filenames '*.cfc'
+
+      def self.keywords
+        @keywords ||= %w(
+          if else var xml default break switch do try catch throw in continue for return while required
+        )
+      end
+
+      def self.declarations
+        @declarations ||= %w(
+          component property function remote public package private
+        )
+      end
+
+      def self.types
+        @types ||= %w(
+          any array binary boolean component date guid numeric query string struct uuid void xml
+        )
+      end
+
+      constants = %w(application session client cookie super this variables arguments cgi)
+
+
+      operators = %w(\+\+ -- && \|\| <= >= < > == != mod eq lt gt lte gte not is and or xor eqv imp equal contains \? )
+      dotted_id = /[$a-zA-Z_][a-zA-Z0-9_.]*/
+
+      state :root do
+        mixin :comments_and_whitespace
+        rule /(?:#{operators.join('|')}|does not contain|greater than(?: or equal to)?|less than(?: or equal to)?)\b/i, Operator, :expr_start
+        rule %r([-<>+*%&|\^/!=]=?), Operator, :expr_start
+
+        rule /[(\[,]/, Punctuation, :expr_start
+        rule /;/, Punctuation, :statement
+        rule /[)\].]/, Punctuation
+
+        rule /[?]/ do
+          token Punctuation
+          push :ternary
+          push :expr_start
+        end
+
+        rule /[{}]/, Punctuation, :statement
+
+        rule /(?:#{constants.join('|')})\b/, Name::Constant
+        rule /(?:true|false|null)\b/, Keyword::Constant
+        rule /import\b/, Keyword::Namespace, :import
+        rule /(#{dotted_id})(\s*)(:)(\s*)/ do
+          groups Name, Text, Punctuation, Text
+          push :expr_start
+        end
+
+        rule /([A-Za-z_$][\w.]*)(\s*)(\()/ do |m|
+          if self.class.keywords.include? m[1]
+            token Keyword, m[1]
+            token Text, m[2]
+            token Punctuation, m[3]
+          else
+            token Name::Function, m[1]
+            token Text, m[2]
+            token Punctuation, m[3]
+          end
+        end
+
+        rule dotted_id do |m|
+          if self.class.declarations.include? m[0]
+            token Keyword::Declaration
+            push :expr_start
+          elsif self.class.keywords.include? m[0]
+            token Keyword
+            push :expr_start
+          elsif self.class.types.include? m[0]
+            token Keyword::Type
+            push :expr_start
+          else
+            token Name::Other
+          end
+        end
+
+        rule /[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?/, Num::Float
+        rule /0x[0-9a-fA-F]+/, Num::Hex
+        rule /[0-9]+/, Num::Integer
+        rule /"(\\\\|\\"|[^"])*"/, Str::Double
+        rule /'(\\\\|\\'|[^'])*'/, Str::Single
+
+      end
+
+      # same as java, broken out
+      state :comments_and_whitespace do
+        rule /\s+/, Text
+        rule %r(//.*?$), Comment::Single
+        rule %r(/\*.*?\*/)m, Comment::Multiline
+      end
+
+      state :expr_start do
+        mixin :comments_and_whitespace
+
+        rule /[{]/, Punctuation, :object
+
+        rule //, Text, :pop!
+      end
+
+      state :statement do
+
+        rule /[{}]/, Punctuation
+
+        mixin :expr_start
+      end
+
+      # object literals
+      state :object do
+        mixin :comments_and_whitespace
+        rule /[}]/ do
+          token Punctuation
+          push :expr_start
+        end
+
+        rule /(#{dotted_id})(\s*)(:)/ do
+          groups Name::Other, Text, Punctuation
+          push :expr_start
+        end
+
+        rule /:/, Punctuation
+        mixin :root
+      end
+
+      # ternary expressions, where <dotted_id>: is not a label!
+      state :ternary do
+        rule /:/ do
+          token Punctuation
+          goto :expr_start
+        end
+
+        mixin :root
+      end
+
+      state :import do
+        rule /\s+/m, Text
+        rule /[a-z0-9_.]+\*?/i, Name::Namespace, :pop!
+      end
+
+    end
+  end
+end