github-linguist 2.0.1 → 2.1.0

data/bin/linguist

@@ -28,7 +28,7 @@ elsif File.file?(path)
   puts "  language:  #{blob.language}"
 
   if blob.large?
-    puts "  blob is to large to be shown"
+    puts "  blob is too large to be shown"
   end
 
   if blob.generated?

data/lib/linguist.rb

@@ -1,5 +1,5 @@
 require 'linguist/blob_helper'
 require 'linguist/language'
 require 'linguist/mime'
-require 'linguist/pathname'
 require 'linguist/repository'
+require 'linguist/samples'

data/lib/linguist/blob_helper.rb

@@ -1,7 +1,7 @@
 require 'linguist/classifier'
 require 'linguist/language'
 require 'linguist/mime'
-require 'linguist/pathname'
+require 'linguist/samples'
 
 require 'charlock_holmes'
 require 'escape_utils'
@@ -12,13 +12,6 @@ module Linguist
   # BlobHelper is a mixin for Blobish classes that respond to "name",
   # "data" and "size" such as Grit::Blob.
   module BlobHelper
-    # Internal: Get a Pathname wrapper for Blob#name
-    #
-    # Returns a Pathname.
-    def pathname
-      Pathname.new(name || "")
-    end
-
     # Public: Get the extname of the path
     #
     # Examples
@@ -28,7 +21,7 @@ module Linguist
     #
     # Returns a String
     def extname
-      pathname.extname
+      File.extname(name)
     end
 
     # Public: Get the actual blob mime type
@@ -40,7 +33,7 @@ module Linguist
     #
     # Returns a mime type String.
     def mime_type
-      @mime_type ||= pathname.mime_type
+      @mime_type ||= Mime.mime_for(extname)
     end
 
     # Public: Get the Content-Type header value
@@ -72,7 +65,7 @@ module Linguist
       elsif name.nil?
         "attachment"
       else
-        "attachment; filename=#{EscapeUtils.escape_url(pathname.basename)}"
+        "attachment; filename=#{EscapeUtils.escape_url(File.basename(name))}"
       end
     end
 
@@ -95,7 +88,7 @@ module Linguist
     #
     # Return true or false
     def binary_mime_type?
-      if mime_type = Mime.lookup_mime_type_for(pathname.extname)
+      if mime_type = Mime.lookup_mime_type_for(extname)
         mime_type.binary?
       end
     end
@@ -136,13 +129,6 @@ module Linguist
       ['.png', '.jpg', '.jpeg', '.gif'].include?(extname)
     end
 
-    # Public: Is the blob a possible drupal php file?
-    #
-    # Return true or false
-    def drupal_extname?
-      ['.module', '.install', '.test', '.inc'].include?(extname)
-    end
-
     # Public: Is the blob likely to have a shebang?
     #
     # Return true or false
@@ -428,10 +414,7 @@ module Linguist
       disambiguate_extension_language ||
 
         # See if there is a Language for the extension
-        pathname.language ||
-
-        # Look for idioms in first line
-        first_line_language ||
+        Language.find_by_filename(name) ||
 
         # Try to detect Language from shebang line
         shebang_language
@@ -446,179 +429,18 @@ module Linguist
 
     # Internal: Disambiguates between multiple language extensions.
     #
-    # Delegates to "guess_EXTENSION_language".
-    #
-    # Please add additional test coverage to
-    # `test/test_blob.rb#test_language` if you add another method.
-    #
     # Returns a Language or nil.
     def disambiguate_extension_language
       if Language.ambiguous?(extname)
-        # name = "guess_#{extname.sub(/^\./, '')}_language"
-        # send(name) if respond_to?(name)
-
-        possible_languages = Language.all.select { |l| l.extensions.include?(extname) }
+        possible_languages = Language.all.select { |l| l.extensions.include?(extname) }.map(&:name)
         if possible_languages.any?
-          if result = Classifier.instance.classify(data, possible_languages).first
-            result[0]
+          if result = Classifier.classify(Samples::DATA, data, possible_languages).first
+            Language[result[0]]
           end
         end
       end
     end
 
-    # Internal: Guess language of .cls files
-    #
-    # Returns a Language.
-    def guess_cls_language
-      if lines.grep(/^(%|\\)/).any?
-        Language['TeX']
-      elsif lines.grep(/^\s*(CLASS|METHOD|INTERFACE).*:\s*/i).any? || lines.grep(/^\s*(USING|DEFINE)/i).any?
-        Language['OpenEdge ABL']
-      elsif lines.grep(/\{$/).any? || lines.grep(/\}$/).any?
-        Language['Apex']
-      elsif lines.grep(/^(\'\*|Attribute|Option|Sub|Private|Protected|Public|Friend)/i).any?
-        Language['Visual Basic']
-      else
-        # The most common language should be the fallback
-        Language['TeX']
-      end
-    end
-
-    # Internal: Guess language of header files (.h).
-    #
-    # Returns a Language.
-    def guess_h_language
-      if lines.grep(/^@(interface|property|private|public|end)/).any?
-        Language['Objective-C']
-      elsif lines.grep(/^class |^\s+(public|protected|private):/).any?
-        Language['C++']
-      else
-        Language['C']
-      end
-    end
-
-    # Internal: Guess language of .m files.
-    #
-    # Objective-C heuristics:
-    # * Keywords  ("#import", "#include", "#ifdef", #define, "@end") or "//" and opening "\*" comments
-    #
-    # Matlab heuristics:
-    # * Leading "function " of "classdef " keyword
-    # * "%" comments
-    #
-    # Note: All "#" keywords, e.g., "#import", are guaranteed to be Objective-C. Because the ampersand
-    # is used to created function handles and anonymous functions in Matlab, most "@" keywords are not
-    # safe heuristics. However, "end" is a reserved term in Matlab and can't be used to create a valid
-    # function handle. Because @end is required to close any @implementation, @property, @interface,
-    # @synthesize, etc. directive in Objective-C, only @end needs to be checked for.
-    #
-    # Returns a Language.
-    def guess_m_language
-      # Objective-C keywords or comments
-      if lines.grep(/^#(import|include|ifdef|define)|@end/).any? || lines.grep(/^\s*\/\//).any? || lines.grep(/^\s*\/\*/).any?
-        Language['Objective-C']
-
-      # Matlab file function or class or comments
-      elsif lines.any? && lines.first.match(/^\s*(function |classdef )/) || lines.grep(/^\s*%/).any?
-        Language['Matlab']
-
-      # Fallback to Objective-C, don't want any Matlab false positives
-      else
-        Language['Objective-C']
-      end
-    end
-
-    # Internal: Guess language of .pl files
-    #
-    # The rules for disambiguation are:
-    #
-    # 1. Many perl files begin with a shebang
-    # 2. Most Prolog source files have a rule somewhere (marked by the :- operator)
-    # 3. Default to Perl, because it is more popular
-    #
-    # Returns a Language.
-    def guess_pl_language
-      if shebang_script == 'perl'
-        Language['Perl']
-      elsif lines.grep(/:-/).any?
-        Language['Prolog']
-      else
-        Language['Perl']
-      end
-    end
-
-    # Internal: Guess language of .r files.
-    #
-    # Returns a Language.
-    def guess_r_language
-      if lines.grep(/(rebol|(:\s+func|make\s+object!|^\s*context)\s*\[)/i).any?
-        Language['Rebol']
-      else
-        Language['R']
-      end
-    end
-
-    # Internal: Guess language of .t files.
-    #
-    # Returns a Language.
-    def guess_t_language
-      score = 0
-      score += 1 if lines.grep(/^% /).any?
-      score += data.gsub(/ := /).count
-      score += data.gsub(/proc |procedure |fcn |function /).count
-      score += data.gsub(/var \w+: \w+/).count
-
-      # Tell-tale signs its gotta be Perl
-      if lines.grep(/^(my )?(sub |\$|@|%)\w+/).any?
-        score = 0
-      end
-
-      if score >= 3
-        Language['Turing']
-      else
-        Language['Perl']
-      end
-    end
-
-    # Internal: Guess language of .v files.
-    #
-    # Returns a Language
-    def guess_v_language
-      if lines.grep(/^(\/\*|\/\/|module|parameter|input|output|wire|reg|always|initial|begin|\`)/).any?
-        Language['Verilog']
-      else
-        Language['Coq']
-      end
-    end
-
-    # Internal: Guess language of .gsp files.
-    #
-    # Returns a Language.
-    def guess_gsp_language
-      if lines.grep(/<%|<%@|\$\{|<%|<g:|<meta name="layout"|<r:/).any?
-        Language['Groovy Server Pages']
-      else
-        Language['Gosu']
-      end
-    end
-
-    # Internal: Guess language from the first line.
-    #
-    # Look for leading "<?php" in Drupal files
-    #
-    # Returns a Language.
-    def first_line_language
-      # Only check files with drupal php extensions
-      return unless drupal_extname?
-
-      # Fail fast if blob isn't viewable?
-      return unless viewable?
-
-      if lines.first.to_s =~ /^<\?php/
-        Language['PHP']
-      end
-    end
-
     # Internal: Extract the script name from the shebang line
     #
     # Requires Blob#data
@@ -710,12 +532,5 @@ module Linguist
         ''
       end
     end
-
-    Language.overridden_extensions.each do |extension|
-      name = "guess_#{extension.sub(/^\./, '')}_language".to_sym
-      unless instance_methods.map(&:to_sym).include?(name)
-        raise NotImplementedError, "Language##{name} was not defined"
-      end
-    end
   end
 end

data/lib/linguist/classifier.rb

@@ -1,113 +1,88 @@
-require 'linguist/language'
 require 'linguist/tokenizer'
 
 module Linguist
   # Language bayesian classifier.
   class Classifier
-    # Internal: Path to persisted classifier db.
-    PATH = File.expand_path('../classifier.yml', __FILE__)
-
-    # Public: Check if persisted db exists on disk.
-    #
-    # Returns Boolean.
-    def self.exist?
-      File.exist?(PATH)
-    end
-
-    # Public: Get persisted Classifier instance.
-    #
-    # Returns Classifier.
-    def self.instance
-      @instance ||= YAML.load_file(PATH)
-    end
-
-    # Public: Initialize a Classifier.
-    def initialize
-      @tokens_total    = 0
-      @languages_total = 0
-      @tokens          = Hash.new { |h, k| h[k] = Hash.new(0) }
-      @language_tokens = Hash.new(0)
-      @languages       = Hash.new(0)
-    end
-
-    # Public: Compare Classifier objects.
-    #
-    # other - Classifier object to compare to.
-    #
-    # Returns Boolean.
-    def eql?(other)
-      # Lazy fast check counts only
-      other.is_a?(self.class) &&
-        @tokens_total == other.instance_variable_get(:@tokens_total) &&
-        @languages_total == other.instance_variable_get(:@languages_total)
-    end
-    alias_method :==, :eql?
-
     # Public: Train classifier that data is a certain language.
     #
-    # language - Language of data
+    # db       - Hash classifier database object
+    # language - String language of data
     # data     - String contents of file
     #
     # Examples
     #
-    #   train(Language['Ruby'], "def hello; end")
+    #   Classifier.train(db, 'Ruby', "def hello; end")
     #
     # Returns nothing.
-    def train(language, data)
-      language = language.name
-      tokens   = Tokenizer.new(data).tokens
+    def self.train!(db, language, data)
+      tokens = Tokenizer.tokenize(data)
+
+      db['tokens_total'] ||= 0
+      db['languages_total'] ||= 0
+      db['tokens'] ||= {}
+      db['language_tokens'] ||= {}
+      db['languages'] ||= {}
 
       tokens.each do |token|
-        @tokens[language][token] += 1
-        @language_tokens[language] += 1
-        @tokens_total += 1
+        db['tokens'][language] ||= {}
+        db['tokens'][language][token] ||= 0
+        db['tokens'][language][token] += 1
+        db['language_tokens'][language] ||= 0
+        db['language_tokens'][language] += 1
+        db['tokens_total'] += 1
       end
-      @languages[language] += 1
-      @languages_total += 1
+      db['languages'][language] ||= 0
+      db['languages'][language] += 1
+      db['languages_total'] += 1
 
       nil
     end
 
-    # Public: Verify internal counts are consistent.
+    # Public: Guess language of data.
+    #
+    # db        - Hash of classifer tokens database.
+    # data      - Array of tokens or String data to analyze.
+    # languages - Array of language name Strings to restrict to.
+    #
+    # Examples
     #
-    # Returns Boolean.
-    def verify
-      @languages.inject(0) { |n, (l, c)| n += c } == @languages_total &&
-        @language_tokens.inject(0) { |n, (l, c)| n += c } == @tokens_total &&
-        @tokens.inject(0) { |n, (l, ts)| n += ts.inject(0) { |m, (t, c)| m += c } } == @tokens_total
+    #   Classifier.classify(db, "def hello; end")
+    #   # => [ 'Ruby', 0.90], ['Python', 0.2], ... ]
+    #
+    # Returns sorted Array of result pairs. Each pair contains the
+    # String language name and a Float score.
+    def self.classify(db, tokens, languages = nil)
+      languages ||= db['languages'].keys
+      new(db).classify(tokens, languages)
     end
 
-    # Public: Prune infrequent tokens.
-    #
-    # Returns receiver Classifier instance.
-    def gc
-      self
+    # Internal: Initialize a Classifier.
+    def initialize(db = {})
+      @tokens_total    = db['tokens_total']
+      @languages_total = db['languages_total']
+      @tokens          = db['tokens']
+      @language_tokens = db['language_tokens']
+      @languages       = db['languages']
     end
 
-    # Public: Guess language of data.
+    # Internal: Guess language of data
     #
     # data      - Array of tokens or String data to analyze.
-    # languages - Array of Languages to restrict to.
-    #
-    # Examples
-    #
-    #   classify("def hello; end")
-    #   # => [ [Language['Ruby'], 0.90], [Language['Python'], 0.2], ... ]
+    # languages - Array of language name Strings to restrict to.
     #
     # Returns sorted Array of result pairs. Each pair contains the
-    # Language and a Float score.
-    def classify(tokens, languages = @languages.keys)
+    # String language name and a Float score.
+    def classify(tokens, languages)
       return [] if tokens.nil?
-      tokens = Tokenizer.new(tokens).tokens if tokens.is_a?(String)
+      tokens = Tokenizer.tokenize(tokens) if tokens.is_a?(String)
 
       scores = {}
       languages.each do |language|
-        language_name = language.is_a?(Language) ? language.name : language
-        scores[language_name] = tokens_probability(tokens, language_name) +
-                                   language_probability(language_name)
+        scores[language] = tokens_probability(tokens, language) +
+                                   language_probability(language)
       end
 
-      scores.sort { |a, b| b[1] <=> a[1] }.map { |score| [Language[score[0]], score[1]] }
+      scores.sort { |a, b| b[1] <=> a[1] }.map { |score| [score[0], score[1]] }
     end
 
     # Internal: Probably of set of tokens in a language occuring - P(D | C)
@@ -144,41 +119,5 @@ module Linguist
     def language_probability(language)
       Math.log(@languages[language].to_f / @languages_total.to_f)
     end
-
-    # Public: Serialize classifier to YAML.
-    #
-    # opts - Hash of YAML options.
-    #
-    # Returns nothing.
-    def to_yaml(io)
-      data = "--- !ruby/object:Linguist::Classifier\n"
-
-      data << "languages_total: #{@languages_total}\n"
-      data << "tokens_total: #{@tokens_total}\n"
-
-      data << "languages:\n"
-      @languages.sort.each do |language, count|
-        data << "  #{{language => count}.to_yaml.lines.to_a[1]}"
-      end
-
-      data << "language_tokens:\n"
-      @language_tokens.sort.each do |language, count|
-        data << "  #{{language => count}.to_yaml.lines.to_a[1]}"
-      end
-
-      data << "tokens:\n"
-      @tokens.sort.each do |language, tokens|
-        data << "  #{{language => true}.to_yaml.lines.to_a[1].sub(/ true/, "")}"
-        tokens.sort.each do |token, count|
-          data << "    #{{token => count}.to_yaml.lines.to_a[1]}"
-        end
-      end
-
-      io.write data
-      nil
-    end
   end
-
-  # Eager load instance
-  Classifier.instance if Classifier.exist?
 end