rouge_ecl 0.0.1 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7edfb964a417fdfbb5cf50d7f290da8762f6e971
-  data.tar.gz: 20e938abaa0fef4ad9dd6163de7745374a6b835a
+  metadata.gz: af56629a74d2aa10269cf04e548644d0671c3dbe
+  data.tar.gz: 9c408275361a52873876b46007aaf008d96bac38
 SHA512:
-  metadata.gz: 9d4d3c5df7ab546716df8bfe7ec45bbb1e5702a5a23196d9c15d55cc9edb7985d7debafcd955bb43189a1bdf3b9be0a616237b8b6146bac5f32bab3bad318320
-  data.tar.gz: 5ba4c4a9efa5e7db46f692350d5610d872284c518c326c03dd6e50682f34d7bac66c548def69703c6f2bbb22e3d791ba105f246f4e7e8c7dbc3bc51e5fc5162f
+  metadata.gz: 23f1ca340154e008b0714c48f6f525edc9d728040650ae8cadf2ac784dd78043c8bdc49c3c02daefd0a94d414dc0016b9fb375e4c17cfeb7f458d970c0794ced
+  data.tar.gz: 4f6a04e669eaae23f1c3be31b431b233404b86bb3a50fb4627b52a3440d581f25790f43851cbdb6bf02094744fa6e6769bc778df8213acf1543661ee047042f0

data/Gemfile

@@ -3,12 +3,12 @@ source 'http://rubygems.org'
 gemspec
 
 gem 'bundler', '~> 1.15'
-gem 'rake'
+gem 'rake', '~> 12.0'
 
-gem 'minitest', '>= 5.0'
-gem 'minitest-power_assert'
+gem 'minitest', '~> 4.0'
+gem 'wrong'
 
-gem 'rubocop', '~> 0.49.1'
+gem 'rubocop', '~> 0.49.1' if RUBY_VERSION >= '2.0.0'
 
 # don't try to install redcarpet under jruby
 gem 'redcarpet', :platforms => :ruby

data/lib/rouge.rb

@@ -41,6 +41,13 @@ load load_dir.join('rouge/util.rb')
 load load_dir.join('rouge/text_analyzer.rb')
 load load_dir.join('rouge/token.rb')
 
+load load_dir.join('rouge/guesser.rb')
+load load_dir.join('rouge/guessers/glob_mapping.rb')
+load load_dir.join('rouge/guessers/modeline.rb')
+load load_dir.join('rouge/guessers/filename.rb')
+load load_dir.join('rouge/guessers/mimetype.rb')
+load load_dir.join('rouge/guessers/source.rb')
+
 load load_dir.join('rouge/lexer.rb')
 load load_dir.join('rouge/regex_lexer.rb')
 load load_dir.join('rouge/template_lexer.rb')
@@ -50,15 +57,6 @@ Dir.glob(lexers_dir.join('*.rb')).each do |f|
   Rouge::Lexers.load_lexer(Pathname.new(f).relative_path_from(lexers_dir).to_s)
 end
 
-load load_dir.join('rouge/guesser.rb')
-load load_dir.join('rouge/guessers/util.rb')
-load load_dir.join('rouge/guessers/glob_mapping.rb')
-load load_dir.join('rouge/guessers/modeline.rb')
-load load_dir.join('rouge/guessers/filename.rb')
-load load_dir.join('rouge/guessers/mimetype.rb')
-load load_dir.join('rouge/guessers/source.rb')
-load load_dir.join('rouge/guessers/disambiguation.rb')
-
 load load_dir.join('rouge/formatter.rb')
 load load_dir.join('rouge/formatters/html.rb')
 load load_dir.join('rouge/formatters/html_table.rb')

data/lib/rouge/cli.rb

@@ -13,9 +13,9 @@ module Rouge
     def file
       case input
       when '-'
-        IO.new($stdin.fileno, 'rt:bom|utf-8')
+        IO.new($stdin.fileno, 'r:utf-8')
       when String
-        File.new(input, 'rt:bom|utf-8')
+        File.new(input, 'r:utf-8')
       when ->(i){ i.respond_to? :read }
         input
       end
@@ -44,7 +44,6 @@ module Rouge
       yield %|	help		#{Help.desc}|
       yield %|	style		#{Style.desc}|
       yield %|	list		#{List.desc}|
-      yield %|	guess		#{Guess.desc}|
       yield %|	version		#{Version.desc}|
       yield %||
       yield %|See `rougify help <command>` for more info.|
@@ -98,8 +97,6 @@ module Rouge
         Style
       when 'list'
         List
-      when 'guess'
-        Guess
       end
     end
 
@@ -378,37 +375,6 @@ module Rouge
       end
     end
 
-    class Guess < CLI
-      def self.desc
-        "guess the languages of file"
-      end
-
-      def self.parse(args)
-        new(input_file: args.shift)
-      end
-
-      attr_reader :input_file, :input_source
-
-      def initialize(opts)
-        @input_file = opts[:input_file] || '-'
-        @input_source = FileReader.new(@input_file).read
-      end
-
-      def lexers
-        Lexer.guesses(
-          filename: input_file,
-          source: input_source,
-        )
-      end
-
-      def run
-        lexers.each do |l|
-          puts "{ tag: #{l.tag.inspect}, title: #{l.title.inspect}, desc: #{l.desc.inspect} }"
-        end
-      end
-    end
-
-
   private_class_method
     def self.normalize_syntax(argv)
       out = []

data/lib/rouge/guessers/glob_mapping.rb

@@ -3,8 +3,6 @@ module Rouge
     # This class allows for custom behavior
     # with glob -> lexer name mappings
     class GlobMapping < Guesser
-      include Util
-
       def self.by_pairs(mapping, filename)
         glob_map = {}
         mapping.each do |(glob, lexer_name)|
@@ -31,13 +29,18 @@ module Rouge
 
         collect_best(lexers) do |lexer|
           score = (@glob_map[lexer.name] || []).map do |pattern|
-            if test_glob(pattern, basename)
+            if test_pattern(pattern, basename)
               # specificity is better the fewer wildcards there are
               -pattern.scan(/[*?\[]/).size
             end
           end.compact.min
         end
       end
+
+      private
+      def test_pattern(pattern, path)
+        File.fnmatch?(pattern, path, File::FNM_DOTMATCH | File::FNM_CASEFOLD)
+      end
     end
   end
 end

data/lib/rouge/guessers/modeline.rb

@@ -1,8 +1,6 @@
 module Rouge
   module Guessers
     class Modeline < Guesser
-      include Util
-
       # [jneen] regexen stolen from linguist
       EMACS_MODELINE = /-\*-\s*(?:(?!mode)[\w-]+\s*:\s*(?:[\w+-]+)\s*;?\s*)*(?:mode\s*:)?\s*([\w+-]+)\s*(?:;\s*(?!mode)[\w-]+\s*:\s*[\w+-]+\s*)*;?\s*-\*-/i
 
@@ -27,9 +25,10 @@ module Rouge
         # don't bother reading the stream if we've already decided
         return lexers if lexers.size == 1
 
-        source_text = get_source(@source)
+        source_text = @source
+        source_text = source_text.read if source_text.respond_to? :read
 
-        lines = source_text.split(/\n/)
+        lines = source_text.split(/\r?\n/)
 
         search_space = (lines.first(@lines) + lines.last(@lines)).join("\n")
 

data/lib/rouge/guessers/source.rb

@@ -1,8 +1,6 @@
 module Rouge
   module Guessers
     class Source < Guesser
-      include Util
-
       attr_reader :source
       def initialize(source)
         @source = source
@@ -13,15 +11,27 @@ module Rouge
         # we've already filtered to 1
         return lexers if lexers.size == 1
 
-        source_text = get_source(@source)
+        # If we're filtering against *all* lexers, we only use confident return
+        # values from analyze_text.  But if we've filtered down already, we can trust
+        # the analysis more.
+        threshold = lexers.size < 10 ? 0 : 0.5
+
+        source_text = case @source
+        when String
+          @source
+        when ->(s){ s.respond_to? :read }
+          @source.read
+        else
+          raise 'invalid source'
+        end
 
         Lexer.assert_utf8!(source_text)
 
         source_text = TextAnalyzer.new(source_text)
 
-        collect_best(lexers) do |lexer|
-          next unless lexer.methods(false).include? :detect?
-          lexer.detect?(source_text) ? 1 : nil
+        collect_best(lexers, threshold: threshold) do |lexer|
+          next unless lexer.methods(false).include? :analyze_text
+          lexer.analyze_text(source_text)
         end
       end
     end

data/lib/rouge/lexer.rb

@@ -22,9 +22,7 @@ module Rouge
         new(opts).lex(stream, &b)
       end
 
-      # Given a name in string, return the correct lexer class.
-      # @param [String] name
-      # @return [Class<Rouge::Lexer>,nil]
+      # Given a string, return the correct lexer class.
       def find(name)
         registry[name.to_s]
       end
@@ -44,7 +42,6 @@ module Rouge
       # markdown lexer for highlighting internal code blocks.
       #
       def find_fancy(str, code=nil, additional_options={})
-
         if str && !str.include?('?') && str != 'guess'
           lexer_class = find(str)
           return lexer_class && lexer_class.new(additional_options)
@@ -112,7 +109,7 @@ module Rouge
       def demo(arg=:absent)
         return @demo = arg unless arg == :absent
 
-        @demo = File.read(demo_file, mode: 'rt:bom|utf-8')
+        @demo = File.read(demo_file, encoding: 'utf-8')
       end
 
       # @return a list of all lexers.
@@ -136,7 +133,6 @@ module Rouge
         guessers << Guessers::Filename.new(filename) if filename
         guessers << Guessers::Modeline.new(source) if source
         guessers << Guessers::Source.new(source) if source
-        guessers << Guessers::Disambiguation.new(filename, source) if source && filename
 
         Guesser.guess(guessers, Lexer.all)
       end
@@ -151,23 +147,16 @@ module Rouge
       #   The source itself, which, if guessing by mimetype or filename
       #   fails, will be searched for shebangs, <!DOCTYPE ...> tags, and
       #   other hints.
-      # @param [Proc] fallback called if multiple lexers are detected.
-      #   If omitted, Guesser::Ambiguous is raised.
       #
-      # @see Lexer.detect?
+      # @see Lexer.analyze_text
       # @see Lexer.guesses
-      # @return [Class<Rouge::Lexer>]
-      def guess(info={}, &fallback)
+      def guess(info={})
         lexers = guesses(info)
 
         return Lexers::PlainText if lexers.empty?
         return lexers[0] if lexers.size == 1
 
-        if fallback
-          fallback.call(lexers)
-        else
-          raise Guesser::Ambiguous.new(lexers)
-        end
+        raise Guesser::Ambiguous.new(lexers)
       end
 
       def guess_by_mimetype(mt)
@@ -436,14 +425,16 @@ module Rouge
 
     # @abstract
     #
-    # Return true if there is an in-text indication (such as a shebang
-    # or DOCTYPE declaration) that this lexer should be used.
+    # Return a number between 0 and 1 indicating the likelihood that
+    # the text given should be lexed with this lexer.  The default
+    # implementation returns 0.  Values under 0.5 will only be used
+    # to disambiguate filename or mimetype matches.
     #
     # @param [TextAnalyzer] text
     #   the text to be analyzed, with a couple of handy methods on it,
     #   like {TextAnalyzer#shebang?} and {TextAnalyzer#doctype?}
-    def self.detect?(text)
-      false
+    def self.analyze_text(text)
+      0
     end
   end
 

data/lib/rouge/lexers/apiblueprint.rb

@@ -11,6 +11,10 @@ module Rouge
       filenames '*.apib'
       mimetypes 'text/vnd.apiblueprint'
 
+      def self.analyze_text(text)
+        return 1 if text.start_with?('FORMAT: 1A\n')
+      end
+
       prepend :root do
         # Metadata
         rule(/(\S+)(:\s*)(.*)$/) do

data/lib/rouge/lexers/awk.rb

@@ -10,8 +10,8 @@ module Rouge
       filenames '*.awk'
       mimetypes 'application/x-awk'
 
-      def self.detect?(text)
-        return true if text.shebang?('awk')
+      def self.analyze_text(text)
+        return 1 if text.shebang?('awk')
       end
 
       id = /[$a-zA-Z_][a-zA-Z0-9_]*/

data/lib/rouge/lexers/biml.rb

@@ -8,8 +8,8 @@ module Rouge
       tag 'biml'
       filenames '*.biml'
 
-      def self.detect?(text)
-        return true if text =~ /<\s*Biml\b/
+      def self.analyze_text(text)
+        return 1 if text =~ /<\s*Biml\b/
       end
 
       prepend :root do

data/lib/rouge/lexers/c.rb

@@ -55,6 +55,11 @@ module Rouge
         )
       end
 
+      # high priority for filename matches
+      def self.analyze_text(*)
+        0.3
+      end
+
       def self.builtins
         @builtins ||= []
       end
@@ -85,7 +90,7 @@ module Rouge
 
       state :whitespace do
         rule /\n+/m, Text, :bol
-        rule %r(//(\\.|.)*?$), Comment::Single, :bol
+        rule %r(//(\\.|.)*?\n), Comment::Single, :bol
         mixin :inline_whitespace
       end
 

data/lib/rouge/lexers/coffeescript.rb

@@ -11,8 +11,8 @@ module Rouge
       title "CoffeeScript"
       desc 'The Coffeescript programming language (coffeescript.org)'
 
-      def self.detect?(text)
-        return true if text.shebang? 'coffee'
+      def self.analyze_text(text)
+        return 1 if text.shebang? 'coffee'
       end
 
       def self.keywords

data/lib/rouge/lexers/coq.rb

@@ -8,6 +8,10 @@ module Rouge
       tag 'coq'
       mimetypes 'text/x-coq'
 
+      def self.analyze_text(text)
+        return 0.3 if text.include? "Require"
+      end
+
       def self.gallina
         @gallina ||= Set.new %w(
           as fun if in let match then else return end Type Set Prop

data/lib/rouge/lexers/diff.rb

@@ -9,10 +9,10 @@ module Rouge
       filenames '*.diff', '*.patch'
       mimetypes 'text/x-diff', 'text/x-patch'
 
-      def self.detect?(text)
-        return true if text.start_with?('Index: ')
-        return true if text =~ %r(\Adiff[^\n]*?\ba/[^\n]*\bb/)
-        return true if text =~ /(---|[+][+][+]).*?\n(---|[+][+][+])/
+      def self.analyze_text(text)
+        return 1   if text.start_with?('Index: ')
+        return 1   if text.start_with?('diff ')
+        return 0.9 if text.start_with?('--- ')
       end
 
       state :root do

data/lib/rouge/lexers/digdag.rb

@@ -11,6 +11,10 @@ module Rouge
 
       mimetypes 'application/x-digdag'
 
+      def self.analyze_text(text)
+        # disable YAML.analyze_text
+      end
+
       # http://docs.digdag.io/operators.html
       # as of digdag v0.9.10
       KEYWORD_PATTERN = Regexp.union(%w(

data/lib/rouge/lexers/ecl.rb

@@ -67,50 +67,62 @@ module Rouge
 
       state :root do
         rule /\s+/m, Text
-        rule %r(//(\\.|.)*?$), Comment::Single
-        rule %r(/\*), Comment::Multiline, :multiline_comments
-        rule /\d+/, Num::Integer
-        rule /\b(?i:(abs|acos|aggregate|allnodes|apply|ascii|asin|asstring|atan|atan2|ave|case|catch|choose|choosen|choosesets|clustersize|combine|correlation|cos|cosh|count|covariance|cron|dataset|dedup|define|denormalize|dictionary|distribute|distributed|distribution|ebcdic|enth|error|evaluate|event|eventextra|eventname|exists|exp|failcode|failmessage|fetch|fromunicode|fromxml|getenv|getisvalid|global|graph|group|hash|hash32|hash64|hashcrc|hashmd5|having|httpcall|httpheader|if|iff|index|intformat|isvalid|iterate|join|keyunicode|length|library|limit|ln|local|log|loop|map|matched|matchlength|matchposition|matchtext|matchunicode|max|merge|mergejoin|min|nofold|nolocal|nonempty|normalize|parse|pipe|power|preload|process|project|pull|random|range|rank|ranked|realformat|recordof|regexfind|regexreplace|regroup|rejected|rollup|round|roundup|row|rowdiff|sample|set|sin|sinh|sizeof|soapcall|sort|sorted|sqrt|stepped|stored|sum|table|tan|tanh|thisnode|topn|tounicode|toxml|transfer|transform|trim|truncate|typeof|ungroup|unicodeorder|variance|which|workunit|xmldecode|xmlencode|xmltext|xmlunicode))\b/, Operator
-        rule /\b(?i:(data|string|qstring|varstring|varunicode|unicode))\d+\b/, Keyword::Type
-        rule /\b(?i:(integer|unsigned))[1-8]\b/, Keyword::Type
-        rule /\b(?i:(ascii|big_endian|boolean|data|decimal|ebcdic|grouped|integer|linkcounted|pattern|qstring|real|record|rule|set of|streamed|string|token|udecimal|unicode|unsigned|varstring|varunicode))\b/, Keyword::Type
-        rule /\b(?i)(u?)decimal(\d+(_\d+)?)\b/, Keyword::Type
-        rule /\b(?i:(apply|assert|build|buildindex|evaluate|fail|keydiff|keypatch|loadxml|nothor|notify|output|parallel|sequential|soapcall|wait))\b/, Operator
-        
+        rule %r(//(\\.|.)*?$), Literal::String::Regex
+        rule %r(/\*), Literal::String::Regex, :multiline_comments
+        rule /0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*/, Num::Integer # hex 0x...
+        rule /0b[01]+(?:_[01]+)*/, Num::Integer # binary 0b...
+        rule /0[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*x/, Num::Integer # hex 0...x
+        rule /0[01]+(?:_[01]+)*b/, Num::Integer # binary 0...b
+        rule /\d+\.\d+(e[\+\-]?\d+)?/, Num::Integer # real
+        rule /0_?[0-7]+(?:_[0-7]+)*/, Num::Integer # octal
+        rule /[\d]+(?:_\d+)*/, Num::Integer # integer
+        rule /\b(?i:(all|except|export|group|import|keyed|wild|left|right|self|shared|skip|true|false))\b/, Name::Builtin
+        rule /\b(?i:(abs|acos|aggregate|allnodes|apply|ascii|asin|asstring|atan|atan2|ave|case|catch|choose|choosen|choosesets|clustersize|combine|correlation|cos|cosh|count|covariance|cron|dataset|dedup|define|denormalize|dictionary|distribute|distributed|distribution|ebcdic|enth|error|evaluate|event|eventextra|eventname|exists|exp|failcode|failmessage|fetch|fromunicode|fromxml|getenv|getisvalid|global|graph|group|hash|hash32|hash64|hashcrc|hashmd5|having|httpcall|httpheader|if|iff|index|intformat|isvalid|iterate|join|keyunicode|length|library|limit|ln|local|log|loop|map|matched|matchlength|matchposition|matchtext|matchunicode|max|merge|mergejoin|min|nofold|nolocal|nonempty|normalize|parse|pipe|power|preload|process|project|pull|random|range|rank|ranked|realformat|recordof|regexfind|regexreplace|regroup|rejected|rollup|round|roundup|row|rowdiff|sample|set|sin|sinh|sizeof|soapcall|sort|sorted|sqrt|stepped|stored|sum|table|tan|tanh|thisnode|topn|tounicode|toxml|transfer|transform|trim|truncate|typeof|ungroup|unicodeorder|variance|which|workunit|xmldecode|xmlencode|xmltext|xmlunicode))\b/, Keyword::Type
+        rule /\b(?i:(data|string|qstring|varstring|varunicode|unicode))\d+\b/, Name::Attribute
+        rule /\b(?i:(integer|unsigned))[1-8]\b/, Name::Attribute
+        rule /\b(?i:(ascii|big_endian|boolean|data|decimal|ebcdic|grouped|integer|linkcounted|pattern|qstring|real|record|rule|set of|streamed|string|token|udecimal|unicode|unsigned|varstring|varunicode))\b/, Name::Attribute
+        rule /\b(?i)(u?)decimal(\d+(_\d+)?)\b/, Name::Attribute
+        rule /\b(?i:(apply|assert|build|buildindex|evaluate|fail|keydiff|keypatch|loadxml|nothor|notify|output|parallel|sequential|soapcall|wait))\b/, Name::Class
+        rule /\b(?i:(import|as|checkpoint|deprecated|failcode|failmessage|failure|global|independent|onwarning|persist|priority|recovery|stored|success|wait|when))\b/, Name::Entity
+
         rule /'/, Str::Single, :single_string
         rule /"/, Name::Variable, :double_string
         rule /`/, Name::Variable, :backtick
 
-        rule /(#)(?i)(\w+)/, Operator
+        rule /(#)(?i)(\w+)/, Generic::Subheading
 
         rule /([A-Za-z_]+)\s*(\()/ do |m|
           if !(self.class.keywords.include? m[1].upcase or self.class.builtins.include? m[1].upcase)
-            token Name::Function
+            token Name::Function, m[1]
+            token Punctuation, m[2]
           end
           if self.class.keywords.include? m[1].upcase
-            token Keyword
+            token Generic::Traceback
           end
           if self.class.builtins.include? m[1].upcase
-            token Operator
+            token Name::Builtin
           end
         end
+        rule /[\$\w][\w\d]*/ do |m|
+
+        #rule for all remaining tokens assigning token Name for those not being a keyword
         rule /[\$\w][\w\d]*/ do |m|
           if self.class.keywords.include? m[0].upcase
-            token Keyword
+            token Generic::Traceback
           else
             token Name
           end
         end
         
         rule %r([+*/<>=~!@#%^&|?^-]), Operator
-        rule /[;:()\[\],.]/, Punctuation
+        rule /[;:{}()\[\],.]/, Punctuation
       end
 
       state :multiline_comments do
-        rule %r(/[*]), Comment::Multiline, :multiline_comments
-        rule %r([*]/), Comment::Multiline, :pop!
-        rule %r([^/*]+), Comment::Multiline
-        rule %r([/*]), Comment::Multiline
+        rule %r(/[*]), Literal::String::Regex, :multiline_comments
+        rule %r([*]/), Literal::String::Regex, :pop!
+        rule %r([^/*]+), Literal::String::Regex
+        rule %r([/*]), Literal::String::Regex
       end
 
       state :backtick do