license_auto 0.1.1.1 → 0.1.1.2

data/lib/license_auto/package_manager.rb

@@ -0,0 +1,41 @@
+require 'open-uri'
+require 'find'
+require 'license_auto/logger'
+require 'license_auto/module'
+
+module LicenseAuto
+  class VirtualPackageManager
+
+    virtual :initialize,
+            :parse_dependencies,
+            :dependency_file_pattern
+
+    # def self.package_managers
+    #   [Bundler, NPM, Pip, Bower, Maven, Gradle, CocoaPods, Rebar, Nuget]
+    # end
+
+    # @uri:
+    #   ./filepath/name.txt
+    #   /some/absolute/file/path/name
+    #   http://somesite.com/foo/bar/baz.file
+    def initialize(path)
+      @path = path
+    end
+
+    # @return Array[Dependency]
+    def parse_dependencies; end
+
+    # @return Array[Regexp]
+    def dependency_file_pattern; end
+
+    def dependency_file_path_names(pattern=dependency_file_pattern)
+      if FileTest.directory?(@path)
+        Find.find(@path).select do |filename|
+          FileTest.file?(filename) && filename =~ pattern
+        end
+      else
+        LicenseAuto.logger.fatal("The repo path: #{@path} does not exist!")
+      end
+    end
+  end
+end

data/lib/license_auto/repo.rb

@@ -0,0 +1,79 @@
+require 'hashie/mash'
+require 'license_auto/website/github_com'
+require 'license_auto/package_manager/bundler'
+
+module LicenseAuto
+  class Repo < Hashie::Mash
+
+    # hash example:
+    #     {
+    #         "clone_url": "https://github.com/mineworks/license_auto.git",
+    #         "ref": "readme",
+    #         "access_token": "40 chars token"
+    #     }
+    def initialize(hash)
+      super(hash)
+      @server = chose_repo_server
+      raise("#{hash} is not a Github Repo") unless @server
+    end
+
+    def self.package_managers
+      [Bundler]
+    end
+
+    # @return:
+    # {
+    #     "LicenseAuto::Bundler": [
+    #         {
+    #             "dep_file": "/tmp/license_auto/cache/github.com/mineworks/license_auto.git/Gemfile.lock",
+    #             "deps": [
+    #                 {
+    #                     "name": "addressable",
+    #                     "version": "2.4.0",
+    #                     "remote": "https://rubygems.org/"
+    #                 },
+    #                 {
+    #                     "name": "anemone",
+    #                     "version": "0.7.2",
+    #                     "remote": "https://rubygems.org/"
+    #                 },
+    #                 {
+    #                     "name": "ast",
+    #                     "version": "2.2.0",
+    #                     "remote": "https://rubygems.org/"
+    #                 }
+    #             ]
+    #         }
+    #     ]
+    # }
+    def find_dependencies
+      repo_dir = @server.clone
+      deps = {}
+      Repo.package_managers.each {|pm|
+        # LicenseAuto.logger.debug(pm)
+        items = pm.new(repo_dir).parse_dependencies
+        unless items.empty?
+          deps[pm.to_s] = items
+        end
+      }
+      LicenseAuto.logger.debug(JSON.pretty_generate(deps))
+      deps
+    end
+
+    # def get_ref()
+    #   @server.get_ref(self.ref)
+    # end
+
+    private
+
+    def chose_repo_server
+      source_code_matcher = LicenseAuto::Matcher::SourceURL.new(clone_url)
+      github_matched = source_code_matcher.match_github_resource
+      if github_matched
+        # TODO: pass argument: ref
+
+        @server = GithubCom.new({}, github_matched[:owner], github_matched[:repo], ref=self.ref)
+      end
+    end
+  end
+end

data/lib/license_auto/version.rb

@@ -8,7 +8,7 @@ module LicenseAuto
     MAJOR = 0
     MINOR = 1
     TINY  = 1
-    PRE   = '1'
+    PRE   = '2'
 
     STRING =
         if PRE.empty?

data/lib/license_auto/website/gemfury_com.rb

@@ -0,0 +1,10 @@
+##
+# [Gemfury](https://gemfury.com/) is a hosted repository for your public and private packages,
+# where they are safe and within reach.
+#
+# [Install RubyGems with a Gemfile](https://gemfury.com/help/install-gems#_ga=1.159224378.125260097.1457687192)
+
+class GemfuryCom
+  # TODO:
+
+end

data/lib/license_auto/website/github_com.rb

@@ -0,0 +1,179 @@
+require 'base64'
+
+require 'fileutils'
+require 'github_api'
+require 'git'
+
+
+require 'license_auto/config/config'
+require 'license_auto/license/similarity'
+require 'license_auto/license/readme'
+
+class GithubCom < Website
+
+  HOST = 'github.com'
+  LANGUAGE = nil
+
+  GIT_HASH_LENGTH = 40
+
+  ##
+  # package: Hashie::Mash
+  # user: string
+  # repo: string
+  # ref: string
+  def initialize(package, user, repo, ref=nil)
+    super(package)
+    @ref = ref
+
+    LicenseAuto.logger.debug("#{user}/#{repo}, #{@ref}")
+
+    @server =
+      begin
+        eval('WebMock')
+        LicenseAuto.logger.debug("LicenseAuto under RSpec mode")
+        Github.new(user: user, repo: repo)
+      rescue NameError => e
+        LicenseAuto.logger.debug("LicenseAuto under running mode")
+        basic_auth = "#{LUTO_CONF.github.username}:#{LUTO_CONF.github.access_token}"
+        Github.new(user: user, repo: repo, basic_auth: basic_auth)
+      end
+  end
+
+  ##
+  # @return LicenseInfoWrapper
+
+  def get_license_info()
+    possible_ref = @ref || match_versioned_ref
+    # If possible_ref is nil, the Github API server will return the default branch contents
+    contents = @server.repos.contents.get(path: '/', ref: possible_ref)
+
+    license_files = []
+    readme_files = []
+    notice_files = []
+    contents.each {|obj|
+      if obj.type == 'file'
+        filename_matcher = LicenseAuto::Matcher::FilepathName.new(obj.name)
+        license_files.push(obj) if filename_matcher.match_license_file
+        readme_files.push(obj) if filename_matcher.match_readme_file
+        notice_files.push(obj) if filename_matcher.match_notice_file
+      end
+    }
+
+    license_files = license_files.map {|obj|
+      license_content = get_blobs(obj['sha'])
+      license_name, sim_ratio = LicenseAuto::Similarity.new(license_content).most_license_sim
+      _hash = {
+        name: license_name,
+        sim_ratio: sim_ratio,
+        html_url: obj['html_url'],
+        download_url: obj['download_url'],
+        text: license_content
+      }
+      LicenseAuto::LicenseWrapper.new(_hash)
+    }
+
+    readme_files = readme_files.map {|obj|
+      readme_content = get_blobs(obj['sha'])
+      license_content = LicenseAuto::Readme.new(obj['download_url'], readme_content).license_content
+      LicenseAuto.logger.debug(license_content)
+      if license_content.nil?
+        next
+      else
+        license_name, sim_ratio = LicenseAuto::Similarity.new(license_content).most_license_sim
+        _hash = {
+            name: license_name,
+            sim_ratio: sim_ratio,
+            html_url: obj['html_url'],
+            download_url: obj['download_url'],
+            text: license_content
+        }
+        LicenseAuto::LicenseWrapper.new(_hash)
+      end
+    }.compact!
+
+    LicenseAuto::LicenseInfoWrapper.new(licenses: license_files, readmes: readme_files, notices: notice_files)
+  end
+
+  def get_ref(ref)
+    @server.git_data.references.get(ref: ref)
+  end
+
+  def match_versioned_ref()
+    possible_ref = nil
+    # If provided a Git SHA, use it directly
+    if @package.version.size >= GIT_HASH_LENGTH
+      possible_ref = @package.version
+    else
+      matcher = LicenseAuto::Matcher::FilepathName.new(@package.version)
+      @server.repos.tags do |tag|
+        matched = matcher.match_the_ref(tag.name)
+        if matched
+          possible_ref = tag.name
+          break
+        end
+      end
+    end
+    possible_ref
+  end
+
+  def list_languages
+    langs = @server.repos.languages
+    LicenseAuto.logger.debug("All languaegs: #{langs}")
+    langs
+  end
+
+  def clone
+    info = repo_info
+
+    clone_url = info.body.fetch('clone_url')
+    LicenseAuto.logger.debug(clone_url)
+
+    trimmed_url = clone_url.gsub(/^http[s]?:\/\//, '')
+    clone_dir = "#{LUTO_CACHE_DIR}/#{trimmed_url}"
+    LicenseAuto.logger.debug(clone_dir)
+
+    if Dir.exists?(clone_dir)
+      git = Git.open(clone_dir, :log => LicenseAuto.logger)
+      local_branch = git.branches.local[0].full
+      if local_branch == @ref
+        git.pull(remote='origin', branch=local_branch)
+      else
+        FileUtils::rm_rf(clone_dir)
+        do_clone(clone_url, clone_dir)
+      end
+    else
+      do_clone(clone_url, clone_dir)
+    end
+    clone_dir
+  end
+
+  def do_clone(clone_url, clone_dir)
+    LicenseAuto.logger.debug(@ref)
+    clone_opts = {
+        :depth => 1, # Only last commit history for fast
+        :branch => @ref
+    }
+    LicenseAuto.logger.debug(clone_url)
+    cloned_repo = Git.clone(clone_url, clone_dir, clone_opts)
+  end
+
+  def repo_info
+    @server.repos.get
+  end
+
+  def filter_gitmodules
+  end
+
+  # http://www.rubydoc.info/github/piotrmurach/github/master/Github/Client/GitData/Blobs#get-instance_method
+  def get_blobs(sha)
+    response_wrapper = @server.git_data.blobs.get(@server.user, @server.repo, sha)
+    LicenseAuto.logger.debug(response_wrapper)
+    content = response_wrapper.body.content
+    encoding = response_wrapper.body.encoding
+    if encoding == 'base64'
+      Base64.decode64(content)
+    else
+      LicenseAuto.logger.error("Unknown encoding: #{encoding}")
+    end
+  end
+end

data/lib/license_auto/website/ruby_gems_org.rb

@@ -1,11 +1,15 @@
 require 'gems'
 require 'hashie'
 
-require 'license_auto/website/github'
+require 'license_auto/website/github_com'
+require 'license_auto/exceptions'
 
 
 class RubyGemsOrg < Website
 
+  URI = 'https://rubygems.org/'
+  HOST = 'rubygems.org'
+  LANGUAGE = 'Ruby'
 
   GEM_NOT_FOUND = "This rubygem could not be found."
 
@@ -26,13 +30,16 @@ class RubyGemsOrg < Website
     end
 
     gem_info = get_gem_info
-    gem_info = Hashie::Mash.new(gem_info)
+
+    raise LicenseAuto::PackageNotFound if gem_info.nil?
 
     source_code_matcher = LicenseAuto::Matcher::SourceURL.new(gem_info.source_code_uri)
 
     github_matched = source_code_matcher.match_github_resource
     if github_matched
-      license_info = LicenseAuto::github_get_license_info(github_matched[:owner], github_matched[:repo], @package.version)
+      license_info = GithubCom.new(@package, github_matched[:owner], github_matched[:repo]).get_license_info
+    elsif false
+
     end
 
     # bitbucket_matched = source_code_matcher.match_bitbucket_resource()
@@ -43,7 +50,13 @@ class RubyGemsOrg < Website
 
   def get_gem_info()
     # TODO: Gems.info(@package.name, @package.version)
-    Gems.info(@package.name)
+    gem_info = Gems.info(@package.name)
+    gem_info =
+        if gem_info == GEM_NOT_FOUND
+          nil
+        else
+          Hashie::Mash.new(gem_info)
+        end
   end
 
   # TODO: switch to https://github.com/rubygems/gems/issues/32#issuecomment-195180422

data/lib/license_auto.rb

@@ -1,20 +1,15 @@
 require 'pathname'
-
-module LicenseAuto
-
-end
-
+require 'license_auto/config/config'
+require 'license_auto/logger'
 
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'license_auto'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 
-# puts $LOAD_PATH
-# puts Pathname.new(__FILE__).dirname.join("license_finder")
-
-# require 'license_auto/config/config'
-
 require 'license_auto/website'
 require 'license_auto/matcher'
 require 'license_auto/package'
-require 'license_auto/license_info'
+require 'license_auto/repo'
+require 'license_auto/license/similarity'
+require 'license_auto/license_info_wrapper'
+require 'license_auto/exceptions'
 

data/license_auto.gemspec

@@ -34,11 +34,16 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
+  spec.add_runtime_dependency 'bundler', '~> 1.10', '>= 0'
   spec.add_runtime_dependency "hashie"
+  spec.add_runtime_dependency "log4r"
   spec.add_runtime_dependency "json"
   spec.add_runtime_dependency "gems"
   spec.add_runtime_dependency "github_api"
-
-  spec.add_development_dependency "bundler", "~> 1.10"
+  spec.add_runtime_dependency "tf-idf-similarity"
+  # spec.add_runtime_dependency "github-markup"
+  # spec.add_runtime_dependency "redcarpet"
+  # spec.add_runtime_dependency "rdoc"
+  
   spec.add_development_dependency "rake", "~> 10.0"
 end

data/{lib → remove}/cloner.rb

@@ -10,7 +10,7 @@ module Cloner
   def self.make_path(repo)
     # todo: If repo is a git@github.com:xxx/yyy
     $plog.debug(repo)
-    repo = repo.gsub(/(https:\/\/|http:\/\/|git@)/,'')
+    repo = repo.gsub(/(https:\/\/|http:\/\/|git@)/, '')
     path = "#{AUTO_ROOT}/#{repo}"
   end
 

data/{lib → remove}/parser/golang_parser.rb

@@ -5,7 +5,7 @@ require 'json'
 require 'open3'
 require 'set'
 require_relative '../../lib/cloner'
-require_relative '../../enums'
+require_relative '../../golang'
 require_relative '../../config/config'
 require_relative '../recorder'