license_auto 0.1.1.2 → 0.1.1.3

data/remove/api/spider.rb

@@ -1,47 +0,0 @@
-require "httparty"
-require_relative '../../lib/misc'
-
-module API
-
-  class Spider
-    def initialize(homepage_url, pack_name)
-      @homepage_url = homepage_url
-      @pack_name = pack_name
-      http_proxy = Misc.get_http_proxy
-      @http_option = {}
-      if http_proxy
-        @http_option[:http_proxyaddr] = http_proxy[:addr]
-        @http_option[:http_proxyport] = http_proxy[:port]
-      end
-    end
-
-    def find_source_url()
-      response = HTTParty.get(@homepage_url, options=@http_option)
-      if response.code == 200
-        body = response.licenses
-        # TODO: author name valid
-        pattern = /(http[s]?:\/\/(github\.com|bitbucket\.org)\/|git@(github\.com|bitbucket\.org):)(?<author>.+?)\/#{@pack_name}/i
-        match_result = pattern.match(body)
-        if match_result
-          author = match_result['author']
-          if author != nil
-            source_url = "https://github.com/#{author}/#{@pack_name}"
-            return source_url
-          end
-        end
-      else
-        # TODO: 404
-      end
-    end
-
-  end
-
-end ### API
-
-if __FILE__ == $0
-  url = "http://www.rubyonrails.org"
-  pack_name = 'httparty'
-  s = API::Spider.new(url, pack_name)
-  source_url = s.find_source_url
-  p source_url
-end

data/remove/cloner.rb

@@ -1,154 +0,0 @@
-require 'git'
-require 'fileutils'
-require 'json'
-require_relative '../config/config'
-require_relative '../lib/api/pattern'
-require_relative '../lib/api/github'
-
-module Cloner
-
-  def self.make_path(repo)
-    # todo: If repo is a git@github.com:xxx/yyy
-    $plog.debug(repo)
-    repo = repo.gsub(/(https:\/\/|http:\/\/|git@)/, '')
-    path = "#{AUTO_ROOT}/#{repo}"
-  end
-
-  def self.clone_repo(repo, release_id, repo_id, reclone=false)
-    path = self.make_path(repo)
-    if reclone
-      FileUtils::rm_rf(path)
-    end
-    $plog.debug("Cloning #{repo} into #{path}...")
-    # begin
-      # Git.configure do |config|
-        # If you want to use a custom git binary
-        # config.binary_path = '/git/bin/path'
-
-        # If you need to use a custom SSH script
-        # Config private SSH key on github.com
-      #   config.git_ssh = "#{AUTO_ROOT}/git_ssh_wrapper.sh"
-      # end
-      if Dir.exists?(path)
-        if not $debug
-          g = Git.open(path, :log => $plog)
-          local_branch = g.branches.local[0].full
-          g.pull(remote='origin', branch=local_branch)
-        end
-      else
-        opts = {
-          # :recursive => true
-          # Only last commit history
-          :depth => 1
-        }
-        local_repo = Git.clone(repo, path, opts)
-        path = local_repo.dir.path
-      end
-      $plog.debug("Cloned #{repo} into #{path}.")
-
-      process_gitmodules(path, release_id, repo_id)
-
-      return path
-    # rescue Git::GitExecuteError => e
-    #   $plog.error e
-    #   return nil
-      # return self.clone_repo(repo, reclone=true)
-    # end
-  end
-
-  # todo: Checkout a branch or tag
-  def self.checkout_branch(branch)
-    true
-  end
-
-  def self.process_gitmodules(clone_path, release_id, parent_repo_id)
-    gitmodules = find_gitmodules(clone_path)
-    $plog.info("gitmodules: #{gitmodules}")
-    gitmodules.each {|url|
-      homepage = nil
-      # git@github.com:repo_owner/reop_name
-      ssh_pattern = /^(?<username>.+)@/
-      git_pattern = /^git:\/\//
-      if url =~ ssh_pattern
-        url = url.gsub(/:/, '/').gsub(ssh_pattern, 'https://')
-      elsif url =~ git_pattern
-        url = url.gsub(git_pattern, 'http://')
-      end
-      $plog.debug("gitmodules url: #{url}")
-
-      remote = API::RemoteSourceVCS.new(url)
-      homepage = remote.get_homepage
-      g = remote.vcs
-
-      sub_host, sub_repo_owner, sub_repo_name = g.host, g.owner, g.repo
-      org_url = "#{sub_host}/#{sub_repo_owner}"
-
-      if api_get_whitelist_orgs(org_url).ntuples > 0
-        $plog.debug("whitelist_orgs: #{url}")
-
-        new_added, sub_repo = add_repo(sub_repo_name, url, parent_repo_id=parent_repo_id)
-        sub_repo_id = sub_repo['id'].to_i
-        $plog.debug("sub_repo_id: #{sub_repo_id}, new_added: #{new_added}")
-        case_items = api_query_product_repo(release_id, parent_repo_id)
-        if case_items.ntuples > 0
-          api_add_product_repo(release_id, parent_repo_id, sub_repo_id)
-        end
-
-        if new_added
-          mq_publish_repo(release_id, sub_repo_id)
-        end
-      else
-        $plog.debug("whitelist_orgs not: #{url}")
-        pack_name = sub_repo_name
-        last_commit = g.last_commits
-        pack_version = last_commit ? last_commit['sha'] : nil
-        lang = g.host
-        source_url = g.repo_url
-        license = cmt = nil
-        status = 10
-        add_pack_result = api_add_pack(pack_name, pack_version, lang, homepage, source_url, license, status, cmt)
-        pack_id, is_newbie = add_pack_result['pack_id'].to_i, (add_pack_result['is_newbie'] == 't')
-        r = api_add_product_repo_pack(parent_repo_id, pack_id, release_id)
-        # $plog.debug("r: #{r}")
-        if is_newbie
-          queue_name = 'license_auto.pack'
-          $rmq.publish(queue_name, {:pack_id => pack_id}.to_json, check_exist=true)
-        end
-        $plog.debug("pack_id: #{pack_id}, is_newbie: #{is_newbie}")
-      end
-    }
-  end
-
-  def self.mq_publish_repo(release_id, sub_repo_id)
-    message = {
-      :release_id => release_id,
-      :repo_id => sub_repo_id
-    }
-    queue_name = 'license_auto.repo'
-    $plog.info("submodule is Repo, enqueue MQ.repo, release_id: #{release_id}, sub_repo_id: #{sub_repo_id}")
-    $rmq.publish(queue_name, message.to_json)
-  end
-
-  def self.find_gitmodules(clone_path)
-    gitmodules = []
-    filename = '.gitmodules'
-    file = "#{clone_path}/#{filename}"
-    submodules_exists = File.exists?(file)
-    $plog.debug("submodules_exists: #{submodules_exists}")
-    if submodules_exists
-      contents = File.readlines(file)
-      pattern = /url\s=\s(?<url>.+)(\.git)?$/
-      contents.each {|line|
-        matched = pattern.match(line)
-        if matched
-          $plog.debug("matched: #{matched}, submodule line: #{line}")
-          gitmodules.push(matched[:url].gsub(/\.git$/, ''))
-        end
-      }
-
-    end
-    gitmodules
-  end
-
-end
-

data/remove/db.rb

@@ -1,267 +0,0 @@
-require 'httparty'
-require_relative '../config/config'
-require_relative '../lib/message'
-
-def add_product(product_name)
-  r = $conn.exec_params("select * from product where name = $1", [product_name])
-  if r.ntuples == 1
-    return false, r[0]
-  else
-    product = $conn.exec_params("insert into product (name) select $1 returning *", [product_name])
-    if product.ntuples == 1
-      return true, product[0]
-    end
-  end
-end
-
-def add_repo(repo_name, source_url, parent_repo_id=nil, priv=-1)
-  r = $conn.exec_params("select * from repo where name = $1 and source_url = $2", [repo_name, source_url])
-  if r.ntuples == 1
-    return false, r[0]
-  else
-    repo = $conn.exec_params("insert into repo (name, source_url, priv, parent_repo_id) select $1, $2, $3, $4 returning *",
-                             [repo_name, source_url, priv, parent_repo_id])
-    if repo.ntuples == 1
-      return true, repo[0]
-    end
-  end
-end
-
-def api_get_repo_source_url(repo_id)
-  source_url = nil
-  r = $conn.exec_params("select source_url from repo where id = $1", [repo_id])
-  if r.ntuples == 1
-     source_url = r[0]['source_url']
-  end
-
-  source_url
-end
-
-def api_clear_relations(release_id, repo_id)
-  r = $conn.exec_params("
-    delete from product_repo_pack
-      where id in (
-        select r.id from product_repo_pack r
-          join pack on pack.id = r.pack_id
-          where product_repo_id in (
-           select id from product_repo where
-              release_id = $1
-              and repo_id = $2
-          )
-          and pack.status < 41
-      )", [release_id, repo_id])
-  $plog.warn("You are rerunning the repo's deps, the history relation table data of this repo was deleted")
-end
-
-def api_get_case_by_id(case_id)
-  repo_id = nil
-  r = $conn.exec_params("select product_id, release_id, repo_id from product_repo where id = $1", [case_id])
-  if r.ntuples == 1
-    repo_id = r[0]
-  end
-  repo_id
-end
-
-def api_query_product_repo(release_id, repo_id)
-  r = $conn.exec_params("
-        select * from product_repo
-          where release_id = $1
-          and repo_id = $2", [release_id, repo_id])
-end
-
-def api_get_repo_manifest_file_list(repo_id)
-  r = $conn.exec_params("select ymls from repo where id = $1", [repo_id])
-end
-
-def api_add_product_repo(release_id, parent_repo_id, sub_repo_id)
-  all_products = $conn.exec_params("
-    select * from product_repo
-      where release_id = $1
-      and repo_id = $2", [release_id, parent_repo_id])
-  all_products.each {|p|
-    product_id = p['product_id']
-    begin
-      pg_result = $conn.exec_params("select * from product_repo where release_id = $1 and product_id = $2 and repo_id = $3",
-                          [release_id, product_id, sub_repo_id])
-      if pg_result.ntuples == 0
-        $plog.debug("release_id: #{release_id}, repo_id: #{sub_repo_id}, parent_repo_id: #{parent_repo_id}")
-        r = $conn.exec_params("insert into product_repo (release_id, product_id, repo_id) values ($1, $2, $3)",
-                          [release_id, product_id, sub_repo_id])
-      end
-    rescue Exception => e
-      $plog.fatal(e)
-    end
-  }
-
-end
-
-def api_add_product_repo_pack(repo_id, pack_id, release_id)
-  # $plog.debug("api_add_product_repo_pack: (repo_id=#{repo_id}, pack_id=#{pack_id}, release_id=#{release_id})")
-  r = $conn.exec_params("select add_product_repo_pack($1, $2, $3)", [repo_id, pack_id, release_id])
-  # $plog.debug("SQL: select add_product_repo_pack(#{repo_id}, #{pack_id}, #{release_id})")
-  r[0]
-end
-
-def api_add_pack(pack_name, pack_version, lang, homepage, source_url, license, status, cmt, project_url=nil)
-  # "select * from select add_pack('goose', 'unknown', 'Golang', null, null, null, null, null) as t(pack_id integer, new bool)"
-  # $plog.info("status: #{status}")
-  r = $conn.exec_params("select * from add_pack($1, $2, $3, $4, $5, $6, $7, $8, $9) as t(pack_id integer, is_newbie bool)",
-                        [pack_name, pack_version, lang, homepage, source_url, license, status, cmt, project_url])
-  ret = nil
-  if r.ntuples == 1
-    ret = r[0]
-  end
-  ret
-end
-
-def api_get_complete_ratio(release_id, repo_id)
-  # TODO: port
-  ratio = 0.0
-  api_path = '/api/v1/repo/complete_ratio'
-  api_url = "#{LICENSE_WEBSITE_URL}#{api_path}"
-  response = HTTParty.get(api_url,
-                          :query => {
-                            :release_id => release_id,
-                            :repo_id => repo_id
-                          })
-  if response.code == 200
-    ratio = response.licenses.to_f
-    $plog.debug("complete_ratio: #{ratio}")
-    if ratio >= 0.95
-      release_name = nil
-      repo_source_url = nil
-      content = "Your release: #{release_name}, repo: #{repo_source_url} has completed, check it please."
-      Message.send(content)
-    end
-  else
-    $plog.error("#{response}")
-  end
-  ratio
-end
-
-def api_get_pack_by_id(pack_id)
-  pack = nil
-  r = $conn.exec_params("select id, name, version, source_url, lang, homepage, license, license_text, status from pack
-                           where id = $1", [pack_id])
-  if r.ntuples == 1
-    pack = r[0]
-  end
-  pack
-end
-
-def api_get_std_license_name(where='where 1 = 1')
-  r = $conn.exec("select * from std_license #{where}")
-end
-
-def api_setup_pack_status(pack_id, status, cmt)
-  $plog.debug(cmt)
-  r = $conn.exec_params("update pack set status = $1, cmt = $2, update_at = now() where id = $3", [status, cmt[0..79], pack_id])
-end
-
-def api_setup_case_status(repo_id, status, cmt)
-  $plog.debug(cmt)
-  r = $conn.exec_params("update product_repo set status = $1, cmt = $2, update_at = now() where id = $3", [status, cmt[0..79], repo_id])
-end
-
-
-def api_update_pack_info(pack_id, pack)
-  # r = $conn.exec_params("select update_pack($1,$2,$3,$4,$5,$6,$7,$8,$9)",[pack_id,pack['version'],pack['homepage'],pack['source_url'],pack['license_url'],pack['license'],pack['unclear_license'],pack['license_text'],pack['status']])
-
-
-  r = $conn.exec_params("select update_pack($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)",
-                        [pack_id, pack[:version], pack[:homepage], pack[:source_url], pack[:license_url],
-                         pack[:license], pack[:unclear_license], pack[:license_text], pack[:status], pack[:project_url]])
-  if(r[0] == -1)
-    return false
-  else
-    return true
-  end
-end
-
-def api_get_packs_by_name(name, version, lang)
-  r = $conn.exec_params("select * from pack where name = $1 and version = $2 and lang = $3",[name, version, lang])
-  return r
-end
-
-def api_get_gemdata_by_name(name)
-  r = $gemconn.exec_params("select rubygems.name, versions.number, linksets.home, linksets.code, versions.licenses 
-                              from rubygems, versions, linksets 
-                              where rubygems.id = versions.rubygem_id 
-                              and rubygems.id = linksets.rubygem_id 
-                              and rubygems.name = $1 
-                              and versions.latest = true 
-                              and versions.platform = $2", [name, 'ruby'])
-end
-
-def api_get_gemdata_by_name_and_version(name, version)
-  r = $gemconn.exec_params("select rubygems.name, versions.number, linksets.home, linksets.code, versions.licenses 
-                              from rubygems, versions, linksets 
-                              where rubygems.id = versions.rubygem_id 
-                              and rubygems.id = linksets.rubygem_id 
-                              and rubygems.name = $1 
-                              and versions.number = $2", [name, version])
-end
-
-def api_get_template_result_by_product(name, release_name, release_version)
-  list = nil
-  r = $conn.exec_params("select product.name, repo.name, pack.name, pack.version, pack.unclear_license, pack.license, pack.license_text, pack.source_url 
-                            from product_repo_pack 
-                            join pack on product_repo_pack.pack_id = pack.id 
-                            join product_repo on product_repo_pack.product_repo_id = product_repo.id 
-                            join repo on product_repo.repo_id = repo.id 
-                            join product on product_repo.product_id = product.id 
-                            join release_tbl on product_repo.release_id = release_tbl.id 
-                            where product.name = $1 
-                            and release_tbl.name = $2 
-                            and release_tbl.version = $3", [name, release_name, release_version])
-  if r.ntuples > 0
-    list = r
-  end
-  list
-end
-
-def api_get_repo_list_by_product(name, release_name, release_version)
-  repo_list = nil
-  r = $conn.exec_params("select product_repo.id, repo.name 
-                            from product_repo 
-                            join product on product_repo.product_id = product.id 
-                            join repo on product_repo.repo_id = repo.id 
-                            join release_tbl on product_repo.release_id = release_tbl.id 
-                            where product.name = $1 
-                            and release_tbl.name = $2 
-                            and release_tbl.version = $3", [name, release_name, release_version])
-  if r.ntuples > 0
-    repo_list = r
-  end
-  repo_list
-end
-
-def api_get_template_result_by_product_repo_id(id)
-  list = nil
-  r = $conn.exec_params("select pack.name, pack.version, pack.unclear_license, pack.license, pack.license_text, pack.source_url 
-                            from product_repo_pack 
-                            join pack on product_repo_pack.pack_id = pack.id 
-                            where product_repo_pack.product_repo_id = $1", [id])
-  if r.ntuples > 0
-    list = r
-  end
-  list
-end
-
-def api_get_manifest_download_url(pack_id)
-  r = $conn.exec_params("select source_url from pack where id = $1", [pack_id])
-end
-
-# org_url: github.com/your_org_or_person_name
-def api_get_whitelist_orgs(org_url)
-  r = $conn.exec_params("select * from whitelist_orgs where url = $1", [org_url])
-end
-
-def api_get_repo_by_url(url)
-  r = $conn.exec_params("select * from repo where source_url = $1", [url])
-end
-
-if __FILE__ == $0
-  # p api_get_repo_manifest_file_list(80).values[0]
-  api_get_complete_ratio(1, 2)
-end