github-linguist 4.0.3 → 4.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c83f59f5d07b0f9d0b233205ad068fdce8977d9d
-  data.tar.gz: 255fb8be50d94c14dde66750ebdc262a52538413
+  metadata.gz: 4fd8402379e8ac3de17921cf9831e6db303bea33
+  data.tar.gz: 546482b4f73f6c6a512b0258e0768aeb51fdbfb1
 SHA512:
-  metadata.gz: f3d65aaf387f84f956dffd975d2f5e9869f3f2631a429b1d4819525ffdf2816783a7324d9b66821dedd7080c9f9718b58201fad0d04747fe3a83136151ef547a
-  data.tar.gz: c1f9d19147a6e1ae946dd75222f45ed72e59b0643953c3365c2547a2f7d6847639176aba6c14c69a00cfcca0cd61794d63afefdcc773395efca4c7f2112e533d
+  metadata.gz: 73d9c3e80b0884a3ef96f5281028bf943f183de4f11aecd5ef31b986fb8f13d4fa70a38a475a0ca6ba0e3aa20cd3b8965edd41e5f26f15e35ae375c30974382b
+  data.tar.gz: 18d6610244f861a7c4f925b1332370b39901fb7f640efba8b5a83ca7f80321aa17cf5757dd532b7c54206b1c2b66e99c00aafe2213aefcd695205af7afc01bff

data/lib/linguist.rb

@@ -4,4 +4,5 @@ require 'linguist/heuristics'
 require 'linguist/language'
 require 'linguist/repository'
 require 'linguist/samples'
+require 'linguist/shebang'
 require 'linguist/version'

data/lib/linguist/classifier.rb

@@ -3,6 +3,25 @@ require 'linguist/tokenizer'
 module Linguist
   # Language bayesian classifier.
   class Classifier
+    # Public: Use the classifier to detect language of the blob.
+    #
+    # blob               - An object that quacks like a blob.
+    # possible_languages - Array of Language objects
+    #
+    # Examples
+    #
+    #   Classifier.call(FileBlob.new("path/to/file"), [
+    #     Language["Ruby"], Language["Python"]
+    #   ])
+    #
+    # Returns an Array of Language objects, most probable first.
+    def self.call(blob, possible_languages)
+      language_names = possible_languages.map(&:name)
+      classify(Samples.cache, blob.data, language_names).map do |name, _|
+        Language[name] # Return the actual Language objects
+      end
+    end
+
     # Public: Train classifier that data is a certain language.
     #
     # db       - Hash classifier database object

data/lib/linguist/file_blob.rb

@@ -57,14 +57,20 @@ module Linguist
     #
     # Returns a String.
     def extension
-      # File.extname returns nil if the filename is an extension.
-      extension = File.extname(name)
-      basename = File.basename(name)
-      # Checks if the filename is an extension.
-      if extension.empty? && basename[0] == "."
-        basename
-      else
-        extension
+      extensions.last || ""
+    end
+
+    # Public: Return an array of the file extensions
+    #
+    #     >> Linguist::FileBlob.new("app/views/things/index.html.erb").extensions
+    #     => [".html.erb", ".erb"]
+    #
+    # Returns an Array
+    def extensions
+      basename, *segments = File.basename(name).split(".")
+
+      segments.map.with_index do |segment, index|
+        "." + segments[index..-1].join(".")
       end
     end
   end

data/lib/linguist/heuristics.rb

@@ -1,158 +1,160 @@
 module Linguist
   # A collection of simple heuristics that can be used to better analyze languages.
   class Heuristics
-    ACTIVE = true
-
-    # Public: Given an array of String language names,
-    # apply heuristics against the given data and return an array
-    # of matching languages, or nil.
+    # Public: Use heuristics to detect language of the blob.
+    #
+    # blob               - An object that quacks like a blob.
+    # possible_languages - Array of Language objects
+    #
+    # Examples
     #
-    # data      - Array of tokens or String data to analyze.
-    # languages - Array of language name Strings to restrict to.
+    #   Heuristics.call(FileBlob.new("path/to/file"), [
+    #     Language["Ruby"], Language["Python"]
+    #   ])
     #
-    # Returns an array of Languages or []
-    def self.find_by_heuristics(data, languages)
-      if active?
-        result = []
-
-        if languages.all? { |l| ["Perl", "Prolog"].include?(l) }
-          result = disambiguate_pl(data)
-        end
-        if languages.all? { |l| ["ECL", "Prolog"].include?(l) }
-          result = disambiguate_ecl(data)
-        end
-        if languages.all? { |l| ["IDL", "Prolog"].include?(l) }
-          result = disambiguate_pro(data)
-        end
-        if languages.all? { |l| ["Common Lisp", "OpenCL"].include?(l) }
-          result = disambiguate_cl(data)
-        end
-        if languages.all? { |l| ["Hack", "PHP"].include?(l) }
-          result = disambiguate_hack(data)
-        end
-        if languages.all? { |l| ["Scala", "SuperCollider"].include?(l) }
-          result = disambiguate_sc(data)
-        end
-        if languages.all? { |l| ["AsciiDoc", "AGS Script"].include?(l) }
-          result = disambiguate_asc(data)
-        end
-        if languages.all? { |l| ["FORTRAN", "Forth"].include?(l) }
-          result = disambiguate_f(data)
-        end
-        return result
+    # Returns an Array of languages, or empty if none matched or were inconclusive.
+    def self.call(blob, languages)
+      data = blob.data
+
+      @heuristics.each do |heuristic|
+        return Array(heuristic.call(data)) if heuristic.matches?(languages)
       end
+
+      [] # No heuristics matched
     end
 
-    # .h extensions are ambiguous between C, C++, and Objective-C.
-    # We want to shortcut look for Objective-C _and_ now C++ too!
+    # Internal: Define a new heuristic.
     #
-    # Returns an array of Languages or []
-    def self.disambiguate_c(data)
-      matches = []
-      if data.include?("@interface")
-        matches << Language["Objective-C"]
-      elsif data.include?("#include <cstdint>")
-        matches << Language["C++"]
+    # languages - String names of languages to disambiguate.
+    # heuristic - Block which takes data as an argument and returns a Language or nil.
+    #
+    # Examples
+    #
+    #     disambiguate "Perl", "Prolog" do |data|
+    #       if data.include?("use strict")
+    #         Language["Perl"]
+    #       elsif data.include?(":-")
+    #         Language["Prolog"]
+    #       end
+    #     end
+    #
+    def self.disambiguate(*languages, &heuristic)
+      @heuristics << new(languages, &heuristic)
+    end
+
+    # Internal: Array of defined heuristics
+    @heuristics = []
+
+    # Internal
+    def initialize(languages, &heuristic)
+      @languages = languages
+      @heuristic = heuristic
+    end
+
+    # Internal: Check if this heuristic matches the candidate languages.
+    def matches?(candidates)
+      candidates.all? { |l| @languages.include?(l.name) }
+    end
+
+    # Internal: Perform the heuristic
+    def call(data)
+      @heuristic.call(data)
+    end
+
+    disambiguate "Objective-C", "C++", "C" do |data|
+      if (/@(interface|class|protocol|property|end|synchronised|selector|implementation)\b/.match(data))
+        Language["Objective-C"]
+      elsif (/^\s*#\s*include <(cstdint|string|vector|map|list|array|bitset|queue|stack|forward_list|unordered_map|unordered_set|(i|o|io)stream)>/.match(data) ||
+        /^\s*template\s*</.match(data) || /^[^@]class\s+\w+/.match(data) || /^[^@](private|public|protected):$/.match(data) || /std::.+$/.match(data))
+        Language["C++"]
       end
-      matches
     end
 
-    def self.disambiguate_pl(data)
-      matches = []
-      if data.include?("use strict")
-        matches << Language["Perl"]
+    disambiguate "Perl", "Perl6", "Prolog" do |data|
+      if data.include?("use v6")
+        Language["Perl6"]
+      elsif data.include?("use strict")
+        Language["Perl"]
       elsif data.include?(":-")
-        matches << Language["Prolog"]
+        Language["Prolog"]
       end
-      matches
     end
 
-    def self.disambiguate_ecl(data)
-      matches = []
+    disambiguate "ECL", "Prolog" do |data|
       if data.include?(":-")
-        matches << Language["Prolog"]
+        Language["Prolog"]
       elsif data.include?(":=")
-        matches << Language["ECL"]
+        Language["ECL"]
       end
-      matches
     end
 
-    def self.disambiguate_pro(data)
-      matches = []
-      if (data.include?(":-"))
-        matches << Language["Prolog"]
-      else
-        matches << Language["IDL"]
-      end
-      matches
-    end
-
-    def self.disambiguate_ts(data)
-      matches = []
-      if (data.include?("</translation>"))
-        matches << Language["XML"]
+    disambiguate "IDL", "Prolog" do |data|
+      if data.include?(":-")
+        Language["Prolog"]
       else
-        matches << Language["TypeScript"]
+        Language["IDL"]
       end
-      matches
     end
 
-    def self.disambiguate_cl(data)
-      matches = []
+    disambiguate "Common Lisp", "OpenCL", "Cool" do |data|
       if data.include?("(defun ")
-        matches << Language["Common Lisp"]
+        Language["Common Lisp"]
+      elsif /^class/x.match(data)
+        Language["Cool"]
       elsif /\/\* |\/\/ |^\}/.match(data)
-        matches << Language["OpenCL"]
+        Language["OpenCL"]
       end
-      matches
-    end
-
-    def self.disambiguate_r(data)
-      matches = []
-      matches << Language["Rebol"] if /\bRebol\b/i.match(data)
-      matches << Language["R"] if data.include?("<-")
-      matches
     end
 
-    def self.disambiguate_hack(data)
-      matches = []
+    disambiguate "Hack", "PHP" do |data|
       if data.include?("<?hh")
-        matches << Language["Hack"]
+        Language["Hack"]
       elsif /<?[^h]/.match(data)
-        matches << Language["PHP"]
+        Language["PHP"]
       end
-      matches
     end
 
-    def self.disambiguate_sc(data)
-      matches = []
-      if (/\^(this|super)\./.match(data) || /^\s*(\+|\*)\s*\w+\s*{/.match(data) || /^\s*~\w+\s*=\./.match(data))
-        matches << Language["SuperCollider"]
-      end
-      if (/^\s*import (scala|java)\./.match(data) || /^\s*val\s+\w+\s*=/.match(data) || /^\s*class\b/.match(data))
-        matches << Language["Scala"]
+    disambiguate "Scala", "SuperCollider" do |data|
+      if /\^(this|super)\./.match(data) || /^\s*(\+|\*)\s*\w+\s*{/.match(data) || /^\s*~\w+\s*=\./.match(data)
+        Language["SuperCollider"]
+      elsif /^\s*import (scala|java)\./.match(data) || /^\s*val\s+\w+\s*=/.match(data) || /^\s*class\b/.match(data)
+        Language["Scala"]
       end
-      matches
     end
 
-    def self.disambiguate_asc(data)
-      matches = []
-      matches << Language["AsciiDoc"] if /^=+(\s|\n)/.match(data)
-      matches
+    disambiguate "AsciiDoc", "AGS Script" do |data|
+      Language["AsciiDoc"] if /^=+(\s|\n)/.match(data)
     end
 
-    def self.disambiguate_f(data)
-      matches = []
+    disambiguate "FORTRAN", "Forth" do |data|
       if /^: /.match(data)
-        matches << Language["Forth"]
+        Language["Forth"]
       elsif /^([c*][^a-z]|      subroutine\s)/i.match(data)
-        matches << Language["FORTRAN"]
+        Language["FORTRAN"]
       end
-      matches
     end
 
-    def self.active?
-      !!ACTIVE
+    disambiguate "F#", "Forth", "GLSL" do |data|
+      if /^(: |new-device)/.match(data)
+        Language["Forth"]
+      elsif /^(#light|import|let|module|namespace|open|type)/.match(data)
+        Language["F#"]
+      elsif /^(#include|#pragma|precision|uniform|varying|void)/.match(data)
+        Language["GLSL"]
+      end
+    end
+
+    disambiguate "Gosu", "JavaScript" do |data|
+      Language["Gosu"] if /^uses java\./.match(data)
+    end
+
+    disambiguate "LoomScript", "LiveScript" do |data|
+      if /^\s*package\s*[\w\.\/\*\s]*\s*{/.match(data)
+        Language["LoomScript"]
+      else
+        Language["LiveScript"]
+      end
     end
+
   end
 end

data/lib/linguist/language.rb

@@ -10,6 +10,8 @@ require 'linguist/heuristics'
 require 'linguist/samples'
 require 'linguist/file_blob'
 require 'linguist/blob_helper'
+require 'linguist/strategy/filename'
+require 'linguist/shebang'
 
 module Linguist
   # Language names that are recognizable by GitHub. Defined languages
@@ -91,6 +93,13 @@ module Linguist
       language
     end
 
+    STRATEGIES = [
+      Linguist::Strategy::Filename,
+      Linguist::Shebang,
+      Linguist::Heuristics,
+      Linguist::Classifier
+    ]
+
     # Public: Detects the Language of the blob.
     #
     # blob - an object that includes the Linguist `BlobHelper` interface;
@@ -98,49 +107,22 @@ module Linguist
     #
     # Returns Language or nil.
     def self.detect(blob)
-      name = blob.name.to_s
-
       # Bail early if the blob is binary or empty.
       return nil if blob.likely_binary? || blob.binary? || blob.empty?
 
-      # A bit of an elegant hack. If the file is executable but extensionless,
-      # append a "magic" extension so it can be classified with other
-      # languages that have shebang scripts.
-      extension = FileBlob.new(name).extension
-      if extension.empty? && blob.mode && (blob.mode.to_i(8) & 05) == 05
-        name += ".script!"
-      end
-
-      # First try to find languages that match based on filename.
-      possible_languages = find_by_filename(name)
-
-      # If there is more than one possible language with that extension (or no
-      # extension at all, in the case of extensionless scripts), we need to continue
-      # our detection work
-      if possible_languages.length > 1
-        data = blob.data
-        possible_language_names = possible_languages.map(&:name)
-        heuristic_languages = Heuristics.find_by_heuristics(data, possible_language_names)
-
-        if heuristic_languages.size > 1
-          possible_language_names = heuristic_languages.map(&:name)
-        end
-
-        # Check if there's a shebang line and use that as authoritative
-        if (result = find_by_shebang(data)) && !result.empty?
-          result.first
-        # No shebang. Still more work to do. Try to find it with our heuristics.
-        elsif heuristic_languages.size == 1
-          heuristic_languages.first
-        # Lastly, fall back to the probabilistic classifier.
-        elsif classified = Classifier.classify(Samples.cache, data, possible_language_names).first
-          # Return the actual Language object based of the string language name (i.e., first element of `#classify`)
-          Language[classified[0]]
+      # Call each strategy until one candidate is returned.
+      STRATEGIES.reduce([]) do |languages, strategy|
+        candidates = strategy.call(blob, languages)
+        if candidates.size == 1
+          return candidates.first
+        elsif candidates.size > 1
+          # More than one candidate was found, pass them to the next strategy.
+          candidates
+        else
+          # No candiates were found, pass on languages from the previous strategy.
+          languages
         end
-      else
-        # Simplest and most common case, we can just return the one match based on extension
-        possible_languages.first
-      end
+      end.first
     end
 
     # Public: Get all Languages
@@ -190,8 +172,13 @@ module Linguist
     # Returns all matching Languages or [] if none were found.
     def self.find_by_filename(filename)
       basename = File.basename(filename)
-      extname = FileBlob.new(filename).extension
-      (@filename_index[basename] + find_by_extension(extname)).compact.uniq
+
+      # find the first extension with language definitions
+      extname = FileBlob.new(filename).extensions.detect do |e|
+        !@extension_index[e].empty?
+      end
+
+      (@filename_index[basename] + @extension_index[extname]).compact.uniq
     end
 
     # Public: Look up Languages by file extension.
@@ -212,20 +199,26 @@ module Linguist
       @extension_index[extname]
     end
 
-    # Public: Look up Languages by shebang line.
+    # DEPRECATED
+    def self.find_by_shebang(data)
+      @interpreter_index[Shebang.interpreter(data)]
+    end
+
+    # Public: Look up Languages by interpreter.
     #
-    # data - Array of tokens or String data to analyze.
+    # interpreter - String of interpreter name
     #
     # Examples
     #
-    #   Language.find_by_shebang("#!/bin/bash\ndate;")
+    #   Language.find_by_interpreter("bash")
     #   # => [#<Language name="Bash">]
     #
     # Returns the matching Language
-    def self.find_by_shebang(data)
-      @interpreter_index[Linguist.interpreter_from_shebang(data)]
+    def self.find_by_interpreter(interpreter)
+      @interpreter_index[interpreter]
     end
 
+
     # Public: Look up Language by its name or lexer.
     #
     # name - The String name of the Language