licensee 5.0.0 → 6.0.0b1

data/lib/licensee/matchers/gemspec_matcher.rb

@@ -1,16 +1,16 @@
 class Licensee
-  class GemspecMatcher < PackageMatcher
+  module Matchers
+    class Gemspec < Package
+      # We definitely don't want to be evaling arbitrary Gemspec files
+      # While not 100% accurate, use some lenient regex to try to grep the
+      # license declaration from the Gemspec as a string, if any
+      LICENSE_REGEX = /^\s*[a-z0-9_]+\.license\s*\=\s*[\'\"]([a-z\-0-9\.]+)[\'\"]\s*$/i
 
-    # We definitely don't want to be evaling arbitrary Gemspec files
-    # While not 100% accurate, use some lenient regex to try to grep the
-    # license declaration from the Gemspec as a string, if any
-    LICENSE_REGEX = /^\s*[a-z0-9_]+\.license\s*\=\s*[\'\"]([a-z\-0-9\.]+)[\'\"]\s*$/i
-
-    private
-
-    def license_property
-      match = file.content.match LICENSE_REGEX
-      match[1].downcase if match && match[1]
+      private
+      def license_property
+        match = @file.content.match LICENSE_REGEX
+        match[1].downcase if match && match[1]
+      end
     end
   end
 end

data/lib/licensee/matchers/npm_bower_matcher.rb

@@ -1,15 +1,15 @@
 class Licensee
-  class NpmBowerMatcher < PackageMatcher
+  module Matchers
+    class NpmBower < Package
+      # While we could parse the package.json or bower.json file, prefer
+      # a lenient regex for speed and security. Moar parsing moar problems.
+      LICENSE_REGEX = /\s*[\"\']license[\"\']\s*\:\s*[\'\"]([a-z\-0-9\.]+)[\'\"],?\s*/i
 
-    # While we could parse the package.json or bower.json file, prefer
-    # a lenient regex for speed and security. Moar parsing moar problems.
-    LICENSE_REGEX = /\s*[\"\']license[\"\']\s*\:\s*[\'\"]([a-z\-0-9\.]+)[\'\"],?\s*/i
-
-    private
-
-    def license_property
-      match = file.content.match LICENSE_REGEX
-      match[1].downcase if match && match[1]
+      private
+      def license_property
+        match = @file.content.match LICENSE_REGEX
+        match[1].downcase if match && match[1]
+      end
     end
   end
 end

data/lib/licensee/matchers/package_matcher.rb

@@ -1,16 +1,17 @@
 class Licensee
-  class PackageMatcher < Matcher
+  module Matchers
+    class Package
+      def initialize(file)
+        @file = file
+      end
 
-    def match
-      Licensee.licenses(:hidden => true).find { |l| l.key == license_property } if file.package?
-    end
-
-    def confidence
-      90
-    end
+      def match
+        Licensee.licenses(:hidden => true).find { |l| l.key == license_property }
+      end
 
-    def self.package_manager?
-      true
+      def confidence
+        90
+      end
     end
   end
 end

data/lib/licensee/project.rb

@@ -1,59 +1,125 @@
+require 'rugged'
+
 class Licensee
+  private
   class Project
-    attr_reader :repository, :revision
-
-    # Initializes a new project
-    #
-    # path_or_repo path to git repo or Rugged::Repository instance
-    # revsion - revision ref, if any
-    def initialize(path_or_repo, revision = nil)
-      if path_or_repo.kind_of? Rugged::Repository
-        @repository = path_or_repo
-      else
-        begin
-          @repository = Rugged::Repository.new(path_or_repo)
-        rescue Rugged::RepositoryError
-          raise if revision
-          @repository = FilesystemRepository.new(path_or_repo)
-        end
-      end
+    def initialize(detect_packages)
+      @detect_packages = detect_packages
+    end
 
-      @revision = revision
+    def detect_packages?
+      @detect_packages
     end
 
     # Returns the matching Licensee::License instance if a license can be detected
     def license
-      @license ||= matched_file.match if matched_file
+      @license ||= matched_file && matched_file.license
+    end
+
+    def matched_file
+      @matched_file ||= (license_file || package_file)
     end
 
     def license_file
       return @license_file if defined? @license_file
-      @license_file = files.select { |f| f.license? }.sort_by { |f| f.license_score }.last
+      @license_file = begin
+        content, name = find_file { |name| LicenseFile.name_score(name) }
+        if content && name
+          LicenseFile.new(content, name)
+        end
+      end
     end
 
     def package_file
-      return unless Licensee.package_manager_files?
+      return unless detect_packages?
       return @package_file if defined? @package_file
-      @package_file = files.select { |f| f.package? }.sort_by { |f| f.package_score }.last
+      @package_file = begin
+        content, name = find_file { |name| PackageInfo.name_score(name) }
+        if content && name 
+          PackageInfo.new(content, name)
+        end
+      end
     end
+  end
 
-    def matched_file
-      return license_file if license_file && license_file.match
-      return package_file if package_file && package_file.match
+  public
+
+  # Git-based project
+  # 
+  # analyze a given git repository for license information
+  class GitProject < Project
+    attr_reader :repository, :revision
+
+    class InvalidRepository < ArgumentError; end
+
+    def initialize(repo, revision: nil, detect_packages: false)
+      if repo.kind_of? Rugged::Repository
+        @repository = repo
+      else
+        @repository = Rugged::Repository.new(repo)
+      end
+
+      @revision = revision
+      super(detect_packages)
+    rescue Rugged::RepositoryError
+      raise InvalidRepository
     end
 
     private
-
     def commit
       @commit ||= revision ? repository.lookup(revision) : repository.last_commit
     end
 
-    def tree
-      @tree ||= commit.tree.select { |blob| blob[:type] == :blob }
+    MAX_LICENSE_SIZE = 64 * 1024
+
+    def load_blob_data(oid)
+      data, _ = Rugged::Blob.to_buffer(repository, oid, MAX_LICENSE_SIZE)
+      data
+    end
+
+    def find_file
+      files = commit.tree.map do |entry|
+        next unless entry[:type] == :blob
+        if (score = yield entry[:name]) > 0
+          { :name => entry[:name], :oid => entry[:oid], :score => score }
+        end
+      end.compact
+
+      return if files.empty?
+      files.sort! { |a, b| b[:score] <=> a[:score] }
+
+      f = files.first
+      [load_blob_data(f[:oid]), f[:name]]
     end
+  end
+
+  # Filesystem-based project
+  #
+  # Analyze a folder on the filesystem for license information
+  class FSProject < Project
+    attr_reader :path
+
+    def initialize(path, detect_packages: false)
+      @path = path
+      super(detect_packages)
+    end
+
+    private
+    def find_file
+      files = [] 
+
+      Dir.foreach(path) do |file|
+        next unless ::File.file?(::File.join(path, file))
+        if (score = yield file) > 0
+          files.push({ :name => file, :score => score })
+        end
+      end
+
+      return if files.empty?
+      files.sort! { |a, b| b[:score] <=> a[:score] }
 
-    def files
-      @files ||= tree.map { |blob| ProjectFile.new(repository.lookup(blob[:oid]), blob[:name]) }
+      f = files.first
+      [::File.read(::File.join(path, f[:name])), f[:name]]
     end
   end
 end

data/lib/licensee/project_file.rb

@@ -1,91 +1,51 @@
+# encoding=utf-8
 class Licensee
-  class ProjectFile
-
-    # Note: File can be a license file (e.g., `LICENSE.txt`)
-    # or a package manager file (e.g, `package.json`)
-
-    attr_reader :blob, :path
-    alias_method :filename, :path
-
-    include Licensee::ContentHelper
-
-    def initialize(blob, path)
-      @blob = blob
-      @path = path
-    end
-
-    # Raw file contents
-    def content
-      @contents ||= blob.content.force_encoding("UTF-8")
-    end
-    alias_method :to_s, :content
-    alias_method :contents, :content
-
-    # File content with all whitespace replaced with a single space
-    def content_normalized
-      @content_normalized ||= normalize_content(content)
-    end
-
-    # Determines which matching strategy to use, returns an instane of that matcher
-    def matcher
-      return @matcher if defined? @matcher
-      @matcher = Licensee.matchers.map { |m| m.new(self) }.find { |m| m.match }
-    end
+  class Project
+    private
+    class File
+      attr_reader :content, :filename
+
+      def initialize(content, filename = nil)
+        @content = content
+        @content.force_encoding(Encoding::UTF_8)
+        @filename = filename
+      end
 
-    # Returns an Licensee::License instance of the matches license
-    def match
-      @match ||= matcher.match if matcher
-    end
+      def matcher
+        @matcher ||= possible_matchers.map { |m| m.new(self) }.find { |m| m.match }
+      end
 
-    # Returns the percent confident with the match
-    def confidence
-      @condience ||= matcher.confidence if matcher
-    end
+      # Returns the percent confident with the match
+      def confidence
+        matcher && matcher.confidence
+      end
 
-    def similarity(other)
-      blob.hashsig(Rugged::Blob::HashSignature::WHITESPACE_SMART)
-      other.hashsig && blob.hashsig ? blob.similarity(other.hashsig) : 0
-    rescue Rugged::InvalidError
-      0
-    end
+      def license
+        matcher && matcher.match
+      end
 
-    def license_score
-      self.class.license_score(filename)
+      alias_method :match, :license
+      alias_method :path, :filename
     end
 
-    def license?
-      license_score != 0.0
-    end
+    public
+    class LicenseFile < File
+      include Licensee::ContentHelper
 
-    def attribution
-      return nil unless license?
-      matches = /^#{CopyrightMatcher::REGEX}$/i.match(content)
-      matches[0].strip if matches
-    end
+      def possible_matchers
+        [Matchers::Copyright, Matchers::Exact, Matchers::Dice]
+      end
 
-    def package_score
-      return 1.0  if filename =~ /[a-zA-Z0-9\-_]+\.gemspec/
-      return 1.0  if filename =~ /package\.json/
-      return 0.75 if filename =~ /bower.json/
-      return 0.0
-    end
+      def wordset
+        @wordset ||= create_word_set(content)
+      end
 
-    def package?
-      Licensee.package_manager_files? && package_score != 0.0
-    end
+      def attribution
+        matches = /^#{Matchers::Copyright::REGEX}$/i.match(content)
+        matches[0].strip if matches
+      end
 
-    class << self
-      # Scores a given file as a potential license
-      #
-      # filename - (string) the name of the file to score
-      #
-      # Returns 1.0 if the file is definitely a license file (e.g, LICENSE)
-      # Returns 0.9 if the file is almost certainly a license file (e.g., LICENSE.md)
-      # Returns 0.8 if the file is probably a license file (e.g., COPYING, COPYING.md)
-      # Returns 0.7 if the file is potentially a license file (e.g., LICENSE.php)
-      # Returns 0.5 if the file is likely a license file (MIT-LICENSE)
-      # Returns 0.0 if the file is definitely not a license file (e.g., index.php)
-      def license_score(filename)
+      def self.name_score(filename)
         return 1.0 if filename =~ /\A(un)?licen[sc]e\z/i
         return 0.9 if filename =~ /\A(un)?licen[sc]e\.(md|markdown|txt)\z/i
         return 0.8 if filename =~ /\Acopy(ing|right)(\.[^.]+)?\z/i
@@ -94,5 +54,25 @@ class Licensee
         return 0.0
       end
     end
+
+    class PackageInfo < File
+      def possible_matchers
+        case ::File.extname(filename)
+        when ".gemspec"
+          [Matchers::Gemspec]
+        when ".json"
+          [Matchers::NpmBower]
+        else
+          []
+        end
+      end
+
+      def self.name_score(filename)
+        return 1.0  if ::File.extname(filename) == ".gemspec"
+        return 1.0  if filename == "package.json"
+        return 0.75 if filename == "bower.json"
+        return 0.0
+      end
+    end
   end
 end

data/lib/licensee/version.rb

@@ -1,3 +1,3 @@
 class Licensee
-  VERSION = "5.0.0"
+  VERSION = "6.0.0b1"
 end

data/licensee.gemspec

@@ -0,0 +1,26 @@
+require File.expand_path("../lib/licensee/version", __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name    = 'licensee'
+  gem.version = Licensee::VERSION
+
+  gem.summary     = "A Ruby Gem to detect under what license a project is distributed"
+  gem.description = "Licensee automates the process of reading LICENSE files and compares their contents to known licenses using a fancy math thing called Rabin-Karp rolling-hashes."
+
+  gem.authors  = ['Ben Balter']
+  gem.email    = 'ben.balter@github.com'
+  gem.homepage = 'http://github.com/benbalter/licensee'
+  gem.license  = "MIT"
+
+  gem.bindir = 'bin'
+  gem.executables << 'licensee'
+
+  gem.add_dependency('rugged', '~> 0.23')
+  gem.add_development_dependency('pry', '~> 0.9')
+  gem.add_development_dependency('shoulda', '~> 3.5')
+  gem.add_development_dependency('rake', '~> 10.3')
+  gem.add_development_dependency('ruby-prof', '~> 0.15')
+
+  # ensure the gem is built out of versioned files
+  gem.files = Dir['Rakefile', '{bin,lib,man,test,vendor,spec}/**/*', 'README*', 'LICENSE*'] & `git ls-files -z`.split("\0")
+end

data/test/fixtures/npm.git/HEAD

@@ -0,0 +1 @@
+ref: refs/heads/master

data/test/fixtures/npm.git/config

@@ -0,0 +1,4 @@
+[core]
+	repositoryformatversion = 0
+	filemode = true
+	bare = true

data/test/fixtures/npm.git/objects/info/packs

@@ -0,0 +1,2 @@
+P pack-03c0879445cabcc37f91d97c7955465adef26f4a.pack
+

data/test/fixtures/npm.git/packed-refs

@@ -0,0 +1,2 @@
+# pack-refs with: peeled fully-peeled 
+0a3da38fcc340e45990520699bc3ec5570e60c11 refs/heads/master