license_auto 0.1.1.2 → 0.1.1.3

data/lib/license_auto/package_manager/npm.rb

@@ -1,2 +1,118 @@
-class Npm
+require 'open3'
+require 'bundler'
+require 'license_auto/package_manager'
+require 'license_auto/website/npm_registry'
+
+module LicenseAuto
+  class Npm < LicenseAuto::PackageManager
+
+    LANGUAGE = 'NodeJS'
+
+    def initialize(path)
+      super(path)
+    end
+
+    def dependency_file_pattern
+      /#{@path}\/(src\/|source\/)?package\.json$/
+    end
+
+    def parse_dependencies
+
+      package_json_files = dependency_file_path_names
+      if package_json_files.empty?
+        LicenseAuto.logger.info("#{LANGUAGE}: #{dependency_file_pattern} file not exist")
+        return []
+      else
+        package_json_files.map {|dep_file|
+          LicenseAuto.logger.debug(dep_file)
+          npm_definition = Hashie::Mash.new(JSON.parse(File.read(dep_file)))
+
+          {
+            dep_file: dep_file,
+            deps: [npm_definition.dependencies, npm_definition.devDependencies].compact.map {|hash|
+                hash.map {|pack_name, semver|
+                  available_version, remote =
+                      if semver =~ /^file:/
+                        # DOC: https://docs.npmjs.com/files/package.json#local-paths
+                        ['local-path', semver]
+                      elsif LicenseAuto::Npm.is_valid_semver?(semver)
+                        pack = Hashie::Mash.new(
+                            "language": "NodeJS",
+                            "name": pack_name,
+                            "group": "",
+                            "version": semver,
+                            "server": "registry.npmjs.org"
+                        )
+                        npm_registry = LicenseAuto::NpmRegistry.new(pack)
+                        version = npm_registry.chose_latest_available_version(semver)
+                        [version, npm_registry.registry]
+                      else
+                        # DOC: https://docs.npmjs.com/files/package.json#git-urls-as-dependencies
+                        git_url = semver.gsub(/^git\+/, '')
+                        matcher = LicenseAuto::Matcher::SourceURL.new(git_url)
+                        github_matched = matcher.match_github_resource
+                        if github_matched
+                          # TODO: ref=github_matched[:ref]
+                          github = GithubCom.new({}, github_matched[:owner], github_matched[:repo])
+                          version = github.list_tags.first.name
+                          LicenseAuto.logger.debug(version)
+                          [version, git_url]
+                        end
+                      end
+                  {
+                      name: pack_name,
+                      version: available_version,
+                      remote: remote
+                  }
+                }
+            }.flatten
+          }
+        }
+      end
+      # LicenseAuto.logger.debug(JSON.pretty_generate(dep_files))
+    end
+
+    # semver: {String}
+    def self.is_valid_semver?(semver)
+      node_cmd = "node -e \"var semver = require('semver'); var valid = semver.validRange('#{semver}'); console.log(valid)\""
+      # LicenseAuto.logger.debug(node_cmd)
+      stdout_str, _stderr_str, _status = Open3.capture3(node_cmd)
+      is_invalid = stdout_str.gsub(/\n/, '') == 'null'
+      # if is_invalid
+      #   LicenseAuto.logger.error("semver: #{semver} is not a valid sem-version")
+      # else
+      #   LicenseAuto.logger.debug("semver: #{semver} is valid")
+      # end
+      not is_invalid
+    end
+
+    # TODO: move
+    def self.check_cli
+      bash_cmd = "node -v"
+      LicenseAuto.logger.debug(bash_cmd)
+      stdout_str, stderr_str, status = Open3.capture3(bash_cmd)
+      node_version = /v5\.\d+\.\d+/
+      if not stderr_str.empty?
+        LicenseAuto.logger.error(stderr_str)
+        return false
+      elsif not stdout_str =~ node_version
+        error = "NodeJS version: #{stdout_str} not satisfied: #{node_version}"
+        LicenseAuto.logger.error(error)
+        return false
+      end
+
+      bash_cmd = "node -e \"const semver = require('semver');\" -r semver"
+      LicenseAuto.logger.debug(bash_cmd)
+      stdout_str, stderr_str, status = Open3.capture3(bash_cmd)
+      unless stderr_str.empty?
+        LicenseAuto.logger.error(stderr_str)
+        LicenseAuto.logger.info(
+            "\nInstall npm packages first using:\n
+              $ npm install\n")
+        return false
+      end
+
+      return true
+    end
+  end
 end

data/lib/license_auto/package_manager/pip.rb

@@ -1,2 +1,8 @@
-class PIP
-end
+# [python-pip](https://pip.pypa.io/en/stable/user_guide/#requirements-files)
+# TODO: use this
+# [](https://requirements-parser.readthedocs.org/en/latest/)
+
+module LicenseAuto
+  class Pip
+  end
+end

data/lib/license_auto/repo.rb

@@ -1,6 +1,11 @@
 require 'hashie/mash'
 require 'license_auto/website/github_com'
 require 'license_auto/package_manager/bundler'
+require 'license_auto/package_manager/npm'
+require 'license_auto/package_manager/golang'
+require 'license_auto/package_manager/gradle'
+require 'license_auto/package_manager/maven'
+require 'license_auto/package_manager/git_module'
 
 module LicenseAuto
   class Repo < Hashie::Mash
@@ -15,10 +20,17 @@ module LicenseAuto
       super(hash)
       @server = chose_repo_server
       raise("#{hash} is not a Github Repo") unless @server
+      @repo_dir = nil
     end
 
     def self.package_managers
-      [Bundler]
+      [
+          Bundler,
+          Npm,
+          Golang,
+          Gradle,
+          Maven
+      ]
     end
 
     # @return:
@@ -44,14 +56,36 @@ module LicenseAuto
     #                 }
     #             ]
     #         }
+    #     ],
+    #     "LicenseAuto::Npm": [
+    #         {
+    #             "dep_file": "/tmp/license_auto/cache/github.com/mineworks/license_auto.git/package.json",
+    #             "deps": [
+    #                 {
+    #                     "name": "bower",
+    #                     "version": "v1.7.9",
+    #                     "remote": "https://github.com/bower/bower.git"
+    #                 },
+    #                 {
+    #                     "name": "lodash.isarguments",
+    #                     "version": "3.0.8",
+    #                     "remote": "http://registry.npmjs.org/"
+    #                 },
+    #                 {
+    #                     "name": "lodash.isarray",
+    #                     "version": "3.0.4",
+    #                     "remote": "http://registry.npmjs.org/"
+    #                 }
+    #             ]
+    #         }
     #     ]
     # }
     def find_dependencies
-      repo_dir = @server.clone
+      @repo_dir = @server.clone
       deps = {}
       Repo.package_managers.each {|pm|
         # LicenseAuto.logger.debug(pm)
-        items = pm.new(repo_dir).parse_dependencies
+        items = pm.new(@repo_dir).parse_dependencies
         unless items.empty?
           deps[pm.to_s] = items
         end
@@ -60,6 +94,20 @@ module LicenseAuto
       deps
     end
 
+
+    # @return Array
+    #
+    def find_git_modules
+      if FileTest.directory?(@repo_dir)
+        pm = LicenseAuto::GitModule.new(@repo_dir)
+        pm.parse_dependencies
+      else
+        error = "Cloned repo_dir is nil"
+        LicenseAuto.logger.error(error)
+        nil
+      end
+    end
+
     # def get_ref()
     #   @server.get_ref(self.ref)
     # end

data/lib/license_auto/source_code_server.rb

@@ -0,0 +1,13 @@
+module LicenseAuto
+  class SourceCodeServer
+    def initialize(source_code_uri)
+      @source_code_uri = source_code_uri
+      @website = match_website(source_code_uri)
+    end
+
+    # @Return a subclass of LicenseAuto::Website
+    def match_website(source_code_uri)
+
+    end
+  end
+end

data/lib/license_auto/var/golang_std_libs.rb

@@ -0,0 +1,160 @@
+##
+# go version go1.6 darwin/amd64
+# $ go list std
+# Doc:
+#   * https://golang.org/src/cmd/go/list.go
+#   * http://pmcgrath.net/how-to-get-golang-package-import-list/
+
+GOLANG_STD_LIBS = [
+    'archive/tar',
+    'archive/zip',
+    'bufio',
+    'bytes',
+    'compress/bzip2',
+    'compress/flate',
+    'compress/gzip',
+    'compress/lzw',
+    'compress/zlib',
+    'container/heap',
+    'container/list',
+    'container/ring',
+    'crypto',
+    'crypto/aes',
+    'crypto/cipher',
+    'crypto/des',
+    'crypto/dsa',
+    'crypto/ecdsa',
+    'crypto/elliptic',
+    'crypto/hmac',
+    'crypto/md5',
+    'crypto/rand',
+    'crypto/rc4',
+    'crypto/rsa',
+    'crypto/sha1',
+    'crypto/sha256',
+    'crypto/sha512',
+    'crypto/subtle',
+    'crypto/tls',
+    'crypto/x509',
+    'crypto/x509/pkix',
+    'database/sql',
+    'database/sql/driver',
+    'debug/dwarf',
+    'debug/elf',
+    'debug/gosym',
+    'debug/macho',
+    'debug/pe',
+    'debug/plan9obj',
+    'encoding',
+    'encoding/ascii85',
+    'encoding/asn1',
+    'encoding/base32',
+    'encoding/base64',
+    'encoding/binary',
+    'encoding/csv',
+    'encoding/gob',
+    'encoding/hex',
+    'encoding/json',
+    'encoding/pem',
+    'encoding/xml',
+    'errors',
+    'expvar',
+    'flag',
+    'fmt',
+    'go/ast',
+    'go/build',
+    'go/constant',
+    'go/doc',
+    'go/format',
+    'go/importer',
+    'go/internal/gccgoimporter',
+    'go/internal/gcimporter',
+    'go/parser',
+    'go/printer',
+    'go/scanner',
+    'go/token',
+    'go/types',
+    'hash',
+    'hash/adler32',
+    'hash/crc32',
+    'hash/crc64',
+    'hash/fnv',
+    'html',
+    'html/template',
+    'image',
+    'image/color',
+    'image/color/palette',
+    'image/draw',
+    'image/gif',
+    'image/internal/imageutil',
+    'image/jpeg',
+    'image/png',
+    'index/suffixarray',
+    'internal/golang.org/x/net/http2/hpack',
+    'internal/race',
+    'internal/singleflight',
+    'internal/testenv',
+    'internal/trace',
+    'io',
+    'io/ioutil',
+    'log',
+    'log/syslog',
+    'math',
+    'math/big',
+    'math/cmplx',
+    'math/rand',
+    'mime',
+    'mime/multipart',
+    'mime/quotedprintable',
+    'net',
+    'net/http',
+    'net/http/cgi',
+    'net/http/cookiejar',
+    'net/http/fcgi',
+    'net/http/httptest',
+    'net/http/httputil',
+    'net/http/internal',
+    'net/http/pprof',
+    'net/internal/socktest',
+    'net/mail',
+    'net/rpc',
+    'net/rpc/jsonrpc',
+    'net/smtp',
+    'net/textproto',
+    'net/url',
+    'os',
+    'os/exec',
+    'os/signal',
+    'os/user',
+    'path',
+    'path/filepath',
+    'reflect',
+    'regexp',
+    'regexp/syntax',
+    'runtime',
+    'runtime/cgo',
+    'runtime/debug',
+    'runtime/internal/atomic',
+    'runtime/internal/sys',
+    'runtime/pprof',
+    'runtime/race',
+    'runtime/trace',
+    'sort',
+    'strconv',
+    'strings',
+    'sync',
+    'sync/atomic',
+    'syscall',
+    'testing',
+    'testing/iotest',
+    'testing/quick',
+    'text/scanner',
+    'text/tabwriter',
+    'text/template',
+    'text/template/parse',
+    'time',
+    'unicode',
+    'unicode/utf16',
+    'unicode/utf8',
+    'unsafe'
+]

data/lib/license_auto/version.rb

@@ -8,7 +8,7 @@ module LicenseAuto
     MAJOR = 0
     MINOR = 1
     TINY  = 1
-    PRE   = '2'
+    PRE   = '3'
 
     STRING =
         if PRE.empty?

data/lib/license_auto/website.rb

@@ -1,5 +1,6 @@
 class Website
 
+  # package: Hashie::Mash
   def initialize(package)
     @package = package
   end

data/lib/license_auto/website/cocoapods_org.rb

@@ -0,0 +1,10 @@
+# https://cocoapods.org/
+
+module LicenseAuto
+  class CocoaPodsOrg
+    def initialize(pod_name, pod_version)
+
+    end
+  end
+
+end

data/lib/license_auto/website/github_com.rb

@@ -16,6 +16,8 @@ class GithubCom < Website
 
   GIT_HASH_LENGTH = 40
 
+  attr_reader :url
+
   ##
   # package: Hashie::Mash
   # user: string
@@ -24,18 +26,18 @@ class GithubCom < Website
   def initialize(package, user, repo, ref=nil)
     super(package)
     @ref = ref
-
-    LicenseAuto.logger.debug("#{user}/#{repo}, #{@ref}")
+    @url = "https://github.com/#{user}/#{repo}"
+    LicenseAuto.logger.debug(@url)
 
     @server =
       begin
         eval('WebMock')
-        LicenseAuto.logger.debug("LicenseAuto under RSpec mode")
+        LicenseAuto.logger.debug("Running LicenseAuto in RSpec mode")
         Github.new(user: user, repo: repo)
       rescue NameError => e
-        LicenseAuto.logger.debug("LicenseAuto under running mode")
+        LicenseAuto.logger.debug("Running LicenseAuto in formal mode")
         basic_auth = "#{LUTO_CONF.github.username}:#{LUTO_CONF.github.access_token}"
-        Github.new(user: user, repo: repo, basic_auth: basic_auth)
+        Github.new(user: user, repo: repo, basic_auth: basic_auth, auto_pagination: true)
       end
   end
 
@@ -44,6 +46,7 @@ class GithubCom < Website
 
   def get_license_info()
     possible_ref = @ref || match_versioned_ref
+    LicenseAuto.logger.debug("possible_ref: #{possible_ref}")
     # If possible_ref is nil, the Github API server will return the default branch contents
     contents = @server.repos.contents.get(path: '/', ref: possible_ref)
 
@@ -62,36 +65,44 @@ class GithubCom < Website
     license_files = license_files.map {|obj|
       license_content = get_blobs(obj['sha'])
       license_name, sim_ratio = LicenseAuto::Similarity.new(license_content).most_license_sim
-      _hash = {
+      LicenseAuto::LicenseWrapper.new(
         name: license_name,
         sim_ratio: sim_ratio,
         html_url: obj['html_url'],
         download_url: obj['download_url'],
         text: license_content
-      }
-      LicenseAuto::LicenseWrapper.new(_hash)
+      )
     }
 
     readme_files = readme_files.map {|obj|
       readme_content = get_blobs(obj['sha'])
       license_content = LicenseAuto::Readme.new(obj['download_url'], readme_content).license_content
-      LicenseAuto.logger.debug(license_content)
+      LicenseAuto.logger.debug("readme_content:\n#{license_content}\n")
       if license_content.nil?
         next
       else
         license_name, sim_ratio = LicenseAuto::Similarity.new(license_content).most_license_sim
-        _hash = {
+        LicenseAuto::LicenseWrapper.new(
             name: license_name,
             sim_ratio: sim_ratio,
             html_url: obj['html_url'],
             download_url: obj['download_url'],
             text: license_content
-        }
-        LicenseAuto::LicenseWrapper.new(_hash)
+        )
       end
-    }.compact!
-
-    LicenseAuto::LicenseInfoWrapper.new(licenses: license_files, readmes: readme_files, notices: notice_files)
+    }.compact
+
+    pack_wrapper = LicenseAuto::PackWrapper.new(
+        homepage: nil,
+        project_url: nil,
+        source_url: @url
+    )
+    LicenseAuto::LicenseInfoWrapper.new(
+        licenses: license_files,
+        readmes: readme_files,
+        notices: notice_files,
+        pack: pack_wrapper
+    )
   end
 
   def get_ref(ref)
@@ -122,6 +133,20 @@ class GithubCom < Website
     langs
   end
 
+  # @return
+  # Array: [#<Hashie::Mash commit=#<Hashie::Mash sha="8065e5c64a22bd6d60e4df8d9be46b5805ec9355" url="https://api.github.com/repos/bower/bower/commits/8065e5c64a22bd6d60e4df8d9be46b5805ec9355"> name="v1.7.9" tarball_url="https://api.github.com/repos/bower/bower/tarball/v1.7.9" zipball_url="https://api.github.com/repos/bower/bower/zipball/v1.7.9">, #<Hashie::Mash commit=
+  def list_tags
+    @server.repos.tags.body
+  end
+
+  def list_commits
+    commits = @server.repos.commits.list
+  end
+
+  def latest_commit
+    latest = list_commits.first
+  end
+
   def clone
     info = repo_info
 
@@ -132,7 +157,7 @@ class GithubCom < Website
     clone_dir = "#{LUTO_CACHE_DIR}/#{trimmed_url}"
     LicenseAuto.logger.debug(clone_dir)
 
-    if Dir.exists?(clone_dir)
+    if Dir.exist?(clone_dir)
       git = Git.open(clone_dir, :log => LicenseAuto.logger)
       local_branch = git.branches.local[0].full
       if local_branch == @ref
@@ -154,7 +179,7 @@ class GithubCom < Website
         :branch => @ref
     }
     LicenseAuto.logger.debug(clone_url)
-    cloned_repo = Git.clone(clone_url, clone_dir, clone_opts)
+    Git.clone(clone_url, clone_dir, clone_opts)
   end
 
   def repo_info
@@ -167,7 +192,7 @@ class GithubCom < Website
   # http://www.rubydoc.info/github/piotrmurach/github/master/Github/Client/GitData/Blobs#get-instance_method
   def get_blobs(sha)
     response_wrapper = @server.git_data.blobs.get(@server.user, @server.repo, sha)
-    LicenseAuto.logger.debug(response_wrapper)
+    # LicenseAuto.logger.debug(response_wrapper)
     content = response_wrapper.body.content
     encoding = response_wrapper.body.encoding
     if encoding == 'base64'