license_auto 0.1.1.1

data/lib/license_auto/package.rb

@@ -0,0 +1,73 @@
+require 'hashie/mash'
+
+require 'license_auto/website/ruby_gems_org'
+
+module LicenseAuto
+
+  ##
+  # Package: {
+  #     language: 'Ruby',                # Ruby|Golang|Java|NodeJS|Erlang|Python|
+  #     name: 'bundler',
+  #     group: 'com.google.http-client', # Optional: Assign nil if your package is not a Java
+  #     version: '1.11.2',               # Optional: Assign nil if check the latest
+  #     project_server: 'rubygems.org'   # Optional: github.com|rubygems.org|pypi.python.org/pypi|registry.npmjs.org
+  # }
+
+  class Package < Hashie::Mash
+    extend LicenseAuto
+
+    ##
+    # Default project website server of all kinds of languages.
+    #
+    # Key: language name
+    #
+    # Value: default project server
+
+    LANGUAGES_PROJECT_SERVER = {
+        Ruby: RubyGemsOrg
+    }
+
+    def initialize(hash)
+      super(hash)
+      @server = nil
+    end
+
+    ##
+    # Class Entry
+
+    def get_license_info(**args)
+
+      # args = {
+      #     fetch_license_text: true
+      # }.merge(args)
+
+      @server.get_license_info if chose_project_server
+
+      # TODO: uncomment these line, add Google or Yahoo!
+      # if @server.nil?
+      #     @server.get_license_info if chose_search_engine
+      # end
+
+    end
+
+    private
+
+    def chose_project_server()
+      begin
+        @server = LANGUAGES_PROJECT_SERVER.fetch(self.language.to_sym).new(self)
+      rescue KeyError => e
+        return nil
+      end
+    end
+
+    def chose_search_engine()
+      # TODO: Website::Google
+      # logger.info("#{self.language} has no adapter. I will google it...")
+      # @search_engine = LicenseAuto::SearchEngine::Google
+      # TODO: Website::Github
+    end
+
+  end
+
+
+end

data/lib/license_auto/version.rb

@@ -0,0 +1,20 @@
+module LicenseAuto
+  # Returns the version of the currently loaded Rails as a <tt>Gem::Version</tt>
+  def self.gem_version
+    Gem::Version.new VERSION::STRING
+  end
+
+  module VERSION
+    MAJOR = 0
+    MINOR = 1
+    TINY  = 1
+    PRE   = '1'
+
+    STRING =
+        if PRE.empty?
+          [MAJOR, MINOR, TINY].compact.join(".")
+        else
+          [MAJOR, MINOR, TINY, PRE].compact.join(".")
+        end
+  end
+end

data/lib/license_auto/website.rb

@@ -0,0 +1,11 @@
+class Website
+
+  def initialize(package)
+    @package = package
+  end
+
+  def to_s
+    @package
+  end
+
+end

data/lib/license_auto/website/github.rb

@@ -0,0 +1,49 @@
+require 'github_api'
+
+module LicenseAuto
+
+  ##
+  #
+  # @owner: string
+  # @repo:  string
+  # @pack_version: string
+  #
+  # @return LicenseInfo
+  GIT_HASH_LENGTH = 40
+
+  def self.github_get_license_info(owner, repo, package_version=nil)
+    github = Github.new(user: owner, repo: repo)
+
+    possible_ref = nil
+    # If provided a Git SHA, use it directly
+    if package_version.size >= GIT_HASH_LENGTH
+      possible_ref = package_version
+    else
+      matcher = LicenseAuto::Matcher::FilepathName.new(package_version)
+      github.repos.tags do |tag|
+        matched = matcher.match_the_ref(tag.name)
+        if matched
+          possible_ref = tag.name
+          break
+        end
+      end
+    end
+
+    # If possible_ref is nil, the Github API server will return the default branch contents
+    contents = github.repos.contents.get(path: '/', ref: possible_ref)
+
+    license_files = []
+    readme_files = []
+    notice_files = []
+    contents.each {|obj|
+      if obj.type == 'file'
+        filename_matcher = LicenseAuto::Matcher::FilepathName.new(obj.name)
+        license_files.push(obj) if filename_matcher.match_license_file
+        readme_files.push(obj) if filename_matcher.match_readme_file
+        notice_files.push(obj) if filename_matcher.match_notice_file
+      end
+    }
+
+    LicenseAuto::LicenseInfo.new(licenses: license_files, readmes: readme_files, notices: notice_files)
+  end
+end

data/lib/license_auto/website/ruby_gems_org.rb

@@ -0,0 +1,79 @@
+require 'gems'
+require 'hashie'
+
+require 'license_auto/website/github'
+
+
+class RubyGemsOrg < Website
+
+
+  GEM_NOT_FOUND = "This rubygem could not be found."
+
+  def initialize(package)
+    super(package)
+  end
+
+  # (Defaults to the latest version if no version is specified.)
+
+  def get_license_info()
+    if @package.version.nil?
+      begin
+        @package.version = get_latest_version['number']
+      rescue Exception => e
+        # TODO: what returned value is better?
+        return nil
+      end
+    end
+
+    gem_info = get_gem_info
+    gem_info = Hashie::Mash.new(gem_info)
+
+    source_code_matcher = LicenseAuto::Matcher::SourceURL.new(gem_info.source_code_uri)
+
+    github_matched = source_code_matcher.match_github_resource
+    if github_matched
+      license_info = LicenseAuto::github_get_license_info(github_matched[:owner], github_matched[:repo], @package.version)
+    end
+
+    # bitbucket_matched = source_code_matcher.match_bitbucket_resource()
+    # if github_matched
+    #   # TODO: bitbucket_matched
+    # end
+  end
+
+  def get_gem_info()
+    # TODO: Gems.info(@package.name, @package.version)
+    Gems.info(@package.name)
+  end
+
+  # TODO: switch to https://github.com/rubygems/gems/issues/32#issuecomment-195180422
+  # @return {
+  #     "authors" => "David Heinemeier Hansson",
+  #     "built_at" => "2016-03-07T00:00:00.000Z",
+  #     "created_at" => "2016-03-07T22:33:22.563Z",
+  #     "description" => "Ruby on Rails is a full-stack web framework optimized for programmer happiness and sustainable productivity. It encourages beautiful code by favoring convention over configuration.",
+  #     "downloads_count" => 28113,
+  #     "metadata" => {},
+  #     "number" => "4.2.6",
+  #     "summary" => "Full-stack web application framework.",
+  #     "platform" => "ruby",
+  #     "ruby_version" => ">= 1.9.3",
+  #     "prerelease" => false,
+  #     "licenses" => [
+  #         [0] "MIT"
+  #     ],
+  #     "requirements" => [],
+  #     "sha" => "a199258c0d2bae09993a6932c49df254fd66428899d1823b8c5285de02e5bc33"
+  # }
+  def get_latest_version()
+    versions = Gems.versions(@package.name).reject { |v| v['prerelease'] }.first
+    if versions == GEM_NOT_FOUND
+      raise(GEM_NOT_FOUND)
+    end
+
+    versions.reject { |v| v['prerelease'] }.first
+  end
+
+  def download_gem()
+  end
+end

data/lib/license_auto/website/ruby_gems_org_db.rb

@@ -0,0 +1,52 @@
+##
+# This file can get PostgreSQL DB dumped data of https://rubygems.org
+# But it's easier to get JSON data from the website API than from the local created database.
+
+require './lib/misc'
+
+
+class RubyGemsOrgDB
+  def initialize
+    @download_prefix = 'https://s3-us-west-2.amazonaws.com/rubygems-dumps/'
+
+    @http_option = {}
+    # TODO: LicenseAuto::Base.config({http_proxy})
+    http_proxy = Misc.get_http_proxy
+    if http_proxy
+      @http_option[:http_proxyaddr] = http_proxy[:addr]
+      @http_option[:http_proxyport] = http_proxy[:port]
+    end
+  end
+
+  def find_archive_download_url()
+    api_url = 'https://s3-us-west-2.amazonaws.com/rubygems-dumps/?prefix=production/public_postgresql'
+    download_url = nil
+
+    response = HTTParty.get(api_url, options=@http_option)
+    if response.code == 200
+      # DOC: http://www.nokogiri.org/tutorials/searching_a_xml_html_document.html#but_i_m_lazy_and_don_t_want_to_deal_with_namespaces_
+      doc = Nokogiri::XML(response.licenses).remove_namespaces!
+
+      xpath = "//Contents[last()]/Key/text()"
+      xpath = "//Contents[last()]/Key/text()"
+      text_node = doc.xpath(xpath)
+      relative_url = text_node.text
+
+      download_url = "#{@download_prefix}#{relative_url}"
+    else
+      raise Exception("http_status: #{response.status}")
+    end
+    download_url
+  end
+
+  # TODO: download function move into misc.rb or others, like launchpad.rb has the same function
+  def download(url)
+    file_pathname = nil
+    return file_pathname
+  end
+
+end
+
+if __FILE__ == $0
+  p RubygemsOrgDB.new.find_archive_download_url
+end

data/lib/license_auto/website/rubydoc_info.rb

@@ -0,0 +1,12 @@
+##
+# http://www.rubydoc.info/
+# * If https://rubygems.org/gems/foo don't provide a source_code link, and you don't want download a gem by link as
+#   https://rubygems.org/downloads/foo-0.0.2.gem and scan its source code, then you can get [YARD](http://yardoc.org)
+#   Ruby documents by this website service.
+#
+# * For safe, download gem package and scan file `license*` is the best way.
+
+class RubyDocInfo
+  # TODO: get Ajax request
+
+end

data/lib/message.rb

@@ -0,0 +1,5 @@
+class Message
+  def self.send(message)
+    # TODO: http api post to website
+  end
+end

data/lib/misc.rb

@@ -0,0 +1,131 @@
+require  'find'
+require 'json'
+require 'httparty'
+
+require 'license_auto/config/config'
+
+module Misc
+
+  def Misc.check_private_repo(where)
+    pg_result = $conn.exec("select * from repo #{where}")
+
+    pg_result.each {|r|
+      source_url = r['source_url']
+      $plog.debug("source_url: #{source_url}")
+      response = HTTParty.get(source_url)
+
+      priv = -1
+      if response.code == 404
+        require_relative './api'
+        g = API::Github.new(source_url)
+        commits = g.list_commits
+        if commits != nil
+          priv = 0
+        end
+      elsif response.code == 200
+        priv = 1
+      end
+      $plog.debug("priv: #{priv}")
+
+      update = $conn.exec_params("update repo set priv = $1 where source_url = $2", [priv, source_url])
+    }
+  end
+
+  # TODO:
+  def Misc.check_git_submodules(where)
+    require_relative './api'
+    # third_party = []
+    # pg_result = $conn.exec("select * from repo #{where}")
+    #
+    # pg_result.each {|r|
+      source_url = 'https://github.com/cloudfoundry-incubator/rep'
+      $plog.debug("source_url: #{source_url}")
+
+      g = API::Github.new(source_url)
+
+      modules = g.get_gitmodules
+      if modules
+        # third_party.push(modules)
+        $plog.debug("modules: #{mmodules}")
+      end
+      # update = $conn.exec_params("update repo set priv = $1 where source_url = $2", [priv, source_url])
+    # }
+  end
+
+  def Misc.enqueue_packs(where)
+    pg_result = $conn.exec("select * from pack #{where}")
+    queue_name = 'license_auto.pack'
+    $plog.debug("enqueue packs no: #{pg_result.ntuples}")
+    pg_result.each {|p|
+      pack_id = p['id'].to_i
+      $rmq.publish(queue_name, {:pack_id => pack_id}.to_json, check_exist=true)
+    }
+  end
+
+  def Misc.enqueue_repos(where, release_id)
+    pg_result = $conn.exec("select * from repo #{where}")
+    queue_name = 'license_auto.repo'
+    $plog.debug("enqueue repos no: #{pg_result.ntuples}")
+    pg_result.each {|repo|
+      repo_id = repo['id'].to_i
+      message = {
+        :repo_id => repo_id,
+        :release_id => release_id
+      }
+      $rmq.publish(queue_name, message.to_json, check_exist=true)
+    }
+  end
+
+  def Misc.get_http_proxy
+    http_proxy = ENV['license_auto_proxy']
+    if http_proxy
+      http_proxy = http_proxy.split(':')
+      return {
+        :addr => http_proxy[0],
+        :port => http_proxy[1]
+      }
+    else
+      return nil
+    end
+  end
+
+  class DirUtils
+    attr_reader :path, :filenames
+    def initialize(path)
+      @path = path
+      @filenames = []
+
+      if File.exists?(path) and File.directory?(path)
+        Find.find(path) do |filename|
+          if File.file?(filename)
+            @filenames.push(filename)
+          end
+        end
+      else
+        raise "Parameter Error: #{path}"
+      end
+    end
+
+    def filter_filename(filename_pattern)
+      result = []
+      @filenames.each {|f|
+        if f =~ filename_pattern
+          result.push(f)
+        end
+      }
+      result
+    end
+  end
+
+end
+
+if __FILE__ == $0
+  # where = ' where 1=1 '
+  # Misc.check_private_repo(where)
+
+  where = ' where status < 30 '
+  Misc.enqueue_packs(where)
+  # Misc.check_private_repo(where)
+  # third_party = Misc.check_git_submodules(where)
+
+end