license_auto 0.1.1.1

data/lib/db.rb

@@ -0,0 +1,267 @@
+require 'httparty'
+require_relative '../config/config'
+require_relative '../lib/message'
+
+def add_product(product_name)
+  r = $conn.exec_params("select * from product where name = $1", [product_name])
+  if r.ntuples == 1
+    return false, r[0]
+  else
+    product = $conn.exec_params("insert into product (name) select $1 returning *", [product_name])
+    if product.ntuples == 1
+      return true, product[0]
+    end
+  end
+end
+
+def add_repo(repo_name, source_url, parent_repo_id=nil, priv=-1)
+  r = $conn.exec_params("select * from repo where name = $1 and source_url = $2", [repo_name, source_url])
+  if r.ntuples == 1
+    return false, r[0]
+  else
+    repo = $conn.exec_params("insert into repo (name, source_url, priv, parent_repo_id) select $1, $2, $3, $4 returning *",
+                             [repo_name, source_url, priv, parent_repo_id])
+    if repo.ntuples == 1
+      return true, repo[0]
+    end
+  end
+end
+
+def api_get_repo_source_url(repo_id)
+  source_url = nil
+  r = $conn.exec_params("select source_url from repo where id = $1", [repo_id])
+  if r.ntuples == 1
+     source_url = r[0]['source_url']
+  end
+
+  source_url
+end
+
+def api_clear_relations(release_id, repo_id)
+  r = $conn.exec_params("
+    delete from product_repo_pack
+      where id in (
+        select r.id from product_repo_pack r
+          join pack on pack.id = r.pack_id
+          where product_repo_id in (
+           select id from product_repo where
+              release_id = $1
+              and repo_id = $2
+          )
+          and pack.status < 41
+      )", [release_id, repo_id])
+  $plog.warn("You are rerunning the repo's deps, the history relation table data of this repo was deleted")
+end
+
+def api_get_case_by_id(case_id)
+  repo_id = nil
+  r = $conn.exec_params("select product_id, release_id, repo_id from product_repo where id = $1", [case_id])
+  if r.ntuples == 1
+    repo_id = r[0]
+  end
+  repo_id
+end
+
+def api_query_product_repo(release_id, repo_id)
+  r = $conn.exec_params("
+        select * from product_repo
+          where release_id = $1
+          and repo_id = $2", [release_id, repo_id])
+end
+
+def api_get_repo_manifest_file_list(repo_id)
+  r = $conn.exec_params("select ymls from repo where id = $1", [repo_id])
+end
+
+def api_add_product_repo(release_id, parent_repo_id, sub_repo_id)
+  all_products = $conn.exec_params("
+    select * from product_repo
+      where release_id = $1
+      and repo_id = $2", [release_id, parent_repo_id])
+  all_products.each {|p|
+    product_id = p['product_id']
+    begin
+      pg_result = $conn.exec_params("select * from product_repo where release_id = $1 and product_id = $2 and repo_id = $3",
+                          [release_id, product_id, sub_repo_id])
+      if pg_result.ntuples == 0
+        $plog.debug("release_id: #{release_id}, repo_id: #{sub_repo_id}, parent_repo_id: #{parent_repo_id}")
+        r = $conn.exec_params("insert into product_repo (release_id, product_id, repo_id) values ($1, $2, $3)",
+                          [release_id, product_id, sub_repo_id])
+      end
+    rescue Exception => e
+      $plog.fatal(e)
+    end
+  }
+
+end
+
+def api_add_product_repo_pack(repo_id, pack_id, release_id)
+  # $plog.debug("api_add_product_repo_pack: (repo_id=#{repo_id}, pack_id=#{pack_id}, release_id=#{release_id})")
+  r = $conn.exec_params("select add_product_repo_pack($1, $2, $3)", [repo_id, pack_id, release_id])
+  # $plog.debug("SQL: select add_product_repo_pack(#{repo_id}, #{pack_id}, #{release_id})")
+  r[0]
+end
+
+def api_add_pack(pack_name, pack_version, lang, homepage, source_url, license, status, cmt, project_url=nil)
+  # "select * from select add_pack('goose', 'unknown', 'Golang', null, null, null, null, null) as t(pack_id integer, new bool)"
+  # $plog.info("status: #{status}")
+  r = $conn.exec_params("select * from add_pack($1, $2, $3, $4, $5, $6, $7, $8, $9) as t(pack_id integer, is_newbie bool)",
+                        [pack_name, pack_version, lang, homepage, source_url, license, status, cmt, project_url])
+  ret = nil
+  if r.ntuples == 1
+    ret = r[0]
+  end
+  ret
+end
+
+def api_get_complete_ratio(release_id, repo_id)
+  # TODO: port
+  ratio = 0.0
+  api_path = '/api/v1/repo/complete_ratio'
+  api_url = "#{LICENSE_WEBSITE_URL}#{api_path}"
+  response = HTTParty.get(api_url,
+                          :query => {
+                            :release_id => release_id,
+                            :repo_id => repo_id
+                          })
+  if response.code == 200
+    ratio = response.licenses.to_f
+    $plog.debug("complete_ratio: #{ratio}")
+    if ratio >= 0.95
+      release_name = nil
+      repo_source_url = nil
+      content = "Your release: #{release_name}, repo: #{repo_source_url} has completed, check it please."
+      Message.send(content)
+    end
+  else
+    $plog.error("#{response}")
+  end
+  ratio
+end
+
+def api_get_pack_by_id(pack_id)
+  pack = nil
+  r = $conn.exec_params("select id, name, version, source_url, lang, homepage, license, license_text, status from pack
+                           where id = $1", [pack_id])
+  if r.ntuples == 1
+    pack = r[0]
+  end
+  pack
+end
+
+def api_get_std_license_name(where='where 1 = 1')
+  r = $conn.exec("select * from std_license #{where}")
+end
+
+def api_setup_pack_status(pack_id, status, cmt)
+  $plog.debug(cmt)
+  r = $conn.exec_params("update pack set status = $1, cmt = $2, update_at = now() where id = $3", [status, cmt[0..79], pack_id])
+end
+
+def api_setup_case_status(repo_id, status, cmt)
+  $plog.debug(cmt)
+  r = $conn.exec_params("update product_repo set status = $1, cmt = $2, update_at = now() where id = $3", [status, cmt[0..79], repo_id])
+end
+
+
+def api_update_pack_info(pack_id, pack)
+  # r = $conn.exec_params("select update_pack($1,$2,$3,$4,$5,$6,$7,$8,$9)",[pack_id,pack['version'],pack['homepage'],pack['source_url'],pack['license_url'],pack['license'],pack['unclear_license'],pack['license_text'],pack['status']])
+
+
+  r = $conn.exec_params("select update_pack($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)",
+                        [pack_id, pack[:version], pack[:homepage], pack[:source_url], pack[:license_url],
+                         pack[:license], pack[:unclear_license], pack[:license_text], pack[:status], pack[:project_url]])
+  if(r[0] == -1)
+    return false
+  else
+    return true
+  end
+end
+
+def api_get_packs_by_name(name, version, lang)
+  r = $conn.exec_params("select * from pack where name = $1 and version = $2 and lang = $3",[name, version, lang])
+  return r
+end
+
+def api_get_gemdata_by_name(name)
+  r = $gemconn.exec_params("select rubygems.name, versions.number, linksets.home, linksets.code, versions.licenses 
+                              from rubygems, versions, linksets 
+                              where rubygems.id = versions.rubygem_id 
+                              and rubygems.id = linksets.rubygem_id 
+                              and rubygems.name = $1 
+                              and versions.latest = true 
+                              and versions.platform = $2", [name, 'ruby'])
+end
+
+def api_get_gemdata_by_name_and_version(name, version)
+  r = $gemconn.exec_params("select rubygems.name, versions.number, linksets.home, linksets.code, versions.licenses 
+                              from rubygems, versions, linksets 
+                              where rubygems.id = versions.rubygem_id 
+                              and rubygems.id = linksets.rubygem_id 
+                              and rubygems.name = $1 
+                              and versions.number = $2", [name, version])
+end
+
+def api_get_template_result_by_product(name, release_name, release_version)
+  list = nil
+  r = $conn.exec_params("select product.name, repo.name, pack.name, pack.version, pack.unclear_license, pack.license, pack.license_text, pack.source_url 
+                            from product_repo_pack 
+                            join pack on product_repo_pack.pack_id = pack.id 
+                            join product_repo on product_repo_pack.product_repo_id = product_repo.id 
+                            join repo on product_repo.repo_id = repo.id 
+                            join product on product_repo.product_id = product.id 
+                            join release_tbl on product_repo.release_id = release_tbl.id 
+                            where product.name = $1 
+                            and release_tbl.name = $2 
+                            and release_tbl.version = $3", [name, release_name, release_version])
+  if r.ntuples > 0
+    list = r
+  end
+  list
+end
+
+def api_get_repo_list_by_product(name, release_name, release_version)
+  repo_list = nil
+  r = $conn.exec_params("select product_repo.id, repo.name 
+                            from product_repo 
+                            join product on product_repo.product_id = product.id 
+                            join repo on product_repo.repo_id = repo.id 
+                            join release_tbl on product_repo.release_id = release_tbl.id 
+                            where product.name = $1 
+                            and release_tbl.name = $2 
+                            and release_tbl.version = $3", [name, release_name, release_version])
+  if r.ntuples > 0
+    repo_list = r
+  end
+  repo_list
+end
+
+def api_get_template_result_by_product_repo_id(id)
+  list = nil
+  r = $conn.exec_params("select pack.name, pack.version, pack.unclear_license, pack.license, pack.license_text, pack.source_url 
+                            from product_repo_pack 
+                            join pack on product_repo_pack.pack_id = pack.id 
+                            where product_repo_pack.product_repo_id = $1", [id])
+  if r.ntuples > 0
+    list = r
+  end
+  list
+end
+
+def api_get_manifest_download_url(pack_id)
+  r = $conn.exec_params("select source_url from pack where id = $1", [pack_id])
+end
+
+# org_url: github.com/your_org_or_person_name
+def api_get_whitelist_orgs(org_url)
+  r = $conn.exec_params("select * from whitelist_orgs where url = $1", [org_url])
+end
+
+def api_get_repo_by_url(url)
+  r = $conn.exec_params("select * from repo where source_url = $1", [url])
+end
+
+if __FILE__ == $0
+  # p api_get_repo_manifest_file_list(80).values[0]
+  api_get_complete_ratio(1, 2)
+end

data/lib/license_auto.rb

@@ -0,0 +1,20 @@
+require 'pathname'
+
+module LicenseAuto
+
+end
+
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'license_auto'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+# puts $LOAD_PATH
+# puts Pathname.new(__FILE__).dirname.join("license_finder")
+
+# require 'license_auto/config/config'
+
+require 'license_auto/website'
+require 'license_auto/matcher'
+require 'license_auto/package'
+require 'license_auto/license_info'
+

data/lib/license_auto/config/config.rb

@@ -0,0 +1,52 @@
+require 'log4r'
+
+$debug = true
+
+AUTO_ROOT = '/tmp/license_auto_cache'
+LAUNCHPAD_SOURCE_DIR = "#{AUTO_ROOT}/lp"
+MANIFEST_SOURCE_DIR = "#{AUTO_ROOT}/manifest"
+if !File.exist?(AUTO_ROOT)
+  Dir.mkdir(AUTO_ROOT)
+end
+if !File.exist?(LAUNCHPAD_SOURCE_DIR)
+  Dir.mkdir(LAUNCHPAD_SOURCE_DIR)
+end
+if !File.exists?(MANIFEST_SOURCE_DIR)
+  Dir.mkdir(MANIFEST_SOURCE_DIR)
+end
+
+STD_LICENSE_DIR = "./extractor_ruby/Package_license"
+
+pf = Log4r::PatternFormatter.new(
+  :pattern => "%d [%l]: %M",
+  :date_format => "%Y/%m/%d %H:%M:%S"
+)
+
+HTTPARTY_DOWNLOAD_TIMEOUT = 480
+
+
+Log4r::StderrOutputter.new('console', :formatter => pf)
+if $debug
+  filename = "auto.log"
+else
+  filename = "#{AUTO_ROOT}/#{rand(1000).to_s}.log"
+end
+
+Log4r::FileOutputter.new('logfile',
+                         :filename => filename,
+                         :trunc => $debug,
+                         :formatter => pf,
+                         :level=>Log4r::DEBUG)
+
+$plog = Log4r::Logger.new('auto.log')
+$plog.add('console', 'logfile')
+
+def log_usage_example()
+  $plog.debug "This is a message with level DEBUG"
+  $plog.info "This is a message with level INFO"
+  $plog.warn "This is a message with level WARN"
+  $plog.error "This is a message with level ERROR"
+  $plog.fatal "This is a message with level FATAL"
+end
+
+LICENSE_WEBSITE_URL = 'http://localhost:3000'

data/lib/license_auto/license/frequency.rb

@@ -0,0 +1,30 @@
+##
+# Statistic
+
+LICENSE_SORTED_FREQUENCY = [
+    'MIT',
+    'Apache2.0',
+    'BSD',
+    'GPL2.0',
+    'RubyClause-6',
+    'LGPL2.1',
+    'GPL3.0',
+    'MPL 2.0',
+    'Python',
+    'LGPL3.0',
+    'PublicDomain',
+    'EPL1.0',
+    'LGPL2.0',
+    'MPL1.1',
+    'CDDL1.0',
+    'OpenSSL',
+    'GPL1.0',
+    'PerlArtistic',
+    'CDDL1.1',
+    'Artistic2.0',
+    'TP-Free',
+    'Artistic1.0',
+    'AFL2.1',
+    'CPL1.0',
+    'Apache1.0',
+]

data/lib/license_auto/license/similarity.rb

@@ -0,0 +1,247 @@
+# TODO: move to units.rb
+class License_recognition
+  def initialize(path = '')
+    @license_text = ''
+    @local_license_list = Array.new
+    @local_license_path = path
+    # TODO: First step, chose the 3 most possible licenses text by keywords; Second step is loop mathing all texts
+    # TODO: Find keywords of license text
+    @sorted_frequency     = ['MIT','MIT2.0','Apache2.0','RubyClause-6','BSD',
+                           'GPL2.0','GPL3.0','LGPL2.1','LGPL3.0']   # Often used license name
+    @license_extension  = ".txt"      # Local license file extensions
+    @similar_list       = Array.new
+    @overload           = 20000       # Text is too long, unable to identify
+    @condition          = 0.85        # Similarity value
+  end
+
+  # description : Find the smallest
+  def min(a, b, c)
+    i = a;
+    i = b if i > b
+    i = c if i > c
+    return i
+  end
+
+  # description : Find the largest
+  def max(a, b, c)
+    i = a;
+    i = b if i < b
+    i = c if i < c
+    return i
+  end
+
+  # description : edit distance
+  def edit_distance(a, b)
+    array = Array.new(2){Array.new(a.size+1)}
+    array[0][0] = 0
+    for i in (1 .. a.size)
+      array[0][i] = i
+    end
+
+    for i in (1 .. b.size)
+      array[i%2][0] = i
+      for j in (1 .. a.size)
+        if b[i - 1] == a[j - 1]
+          array[i%2][j] = array[(i - 1)%2][j - 1]
+        else
+          array[i%2][j] = min(array[i%2][j - 1],array[(i - 1)%2][j - 1],array[(i - 1)%2][j]) + 1
+        end
+      end
+    end
+    return array[i%2][j]
+
+  end
+
+  # description : longest common substring
+  def longest_common_substring(a, b)
+    array = Array.new(2){Array.new(a.size+1)}
+    array[0][0] = 0
+    for i in (1 .. a.size)
+      array[0][i] = 0
+    end
+
+    for i in (1 .. b.size)
+      array[i%2][0] = 0
+      for j in (1 .. a.size)
+        if b[i - 1] == a[j - 1]
+          array[i%2][j] = array[(i - 1)%2][j - 1] + 1
+        else
+          array[i%2][j] = max(array[i%2][j - 1],array[(i - 1)%2][j - 1],array[(i - 1)%2][j])
+        end
+      end
+    end
+    return array[i%2][j]
+  end
+
+  # description : License name list is sorted, commonly used on the front
+  # constant    : License often used list
+  # change      : Waiting list license change
+  def sequence(constant = @sorted_frequency, change = @local_license_list)
+    i = 0
+    for j in (0 ... constant.size)
+      #p constant[j]
+      for k in (i ... change.size)
+        if constant[j] == change[k][1]
+          tmp = change[i]
+          change[i] = change[k]
+          change[k] = tmp
+          i += 1
+          break
+        end
+      end
+    end
+  end
+
+  # description : Get all the local license file path
+  # path        : Local license folder
+  def get_local_license(path = @local_license_path)
+    #p @sorted_frequency
+    if File.directory?(path)
+      Dir.foreach(path) do |file|
+        if file != "." and file != ".." and !File.directory?(file) and File.extname(file) == @license_extension
+          @local_license_list << [File.expand_path(path + '/' + file), File.basename(file,@license_extension)]
+        end
+      end
+    else
+      raise("path: #{path} not found!")
+    end
+    sequence()
+    return @local_license_list
+  end
+
+
+  def sort_insert(data)
+    flag = false
+    if @similar_list.size == 0
+      @similar_list << data
+    else
+      for i in (0 ... @similar_list.size)
+        if data[0] > @similar_list[i][0]
+          @similar_list.insert(i,data)
+          flag = true
+          break
+        end
+      end
+      if false == flag
+        @similar_list << data
+      end
+    end
+  end
+
+  # description : similarity
+  # 0%          : Not the same
+  # 100%        : The same
+  # packge_license : Unrecognized text
+  # path           : local license text
+  def similarity(packge_license, path)
+
+    get_local_license(path)
+
+    package_licen_data = packge_license.scan(/\w+/)
+    # Text is too long, unable to identify, then return null
+    if package_licen_data.size > @overload
+      return nil
+    end
+    local_license_date = Array.new
+    @local_license_list.each do |license|
+      local_license_date.clear
+      local_file = File.readlines(license[0])
+      local_file.each do |line|
+        local_license_date.concat(line.scan(/\w+/))
+      end
+      ed = edit_distance(package_licen_data,local_license_date)
+      lcs = longest_common_substring(package_licen_data,local_license_date)
+      similar = (lcs + 0.0)/(ed + lcs)
+      #p license[1]
+      tmp = [similar, license[1], "ed[#{ed}]", "lcs[#{lcs}]", "web[#{package_licen_data.size}]", "local[#{local_license_date.size}]"]
+      sort_insert(tmp)
+
+      if similar > @condition
+        return license[1]
+      end
+    end
+
+    # p @similar_list
+
+    if @similar_list.size == 0
+      return nil
+    elsif @similar_list[0][0] > 0.76
+      return @similar_list[0][1]
+    elsif @similar_list[0][0] > 0.45
+      return @similar_list[0][1]
+    else
+      return nil
+    end
+  end # def similarity
+
+  def extract_license_text_from_readme(readme)
+    if File.extname(readme['name']) == '.rdoc'
+      regular_start = /^==[ *](copying|copy|license){1}:*/i
+      regular_end   = /^== /
+    elsif File.extname(readme['name']) == '.md'
+      regular_start = /^##[ *](copying|copy|license){1}:*/i
+      regular_end   = /^## /
+    else
+      return nil
+    end
+
+  end
+
+end  # class License_recognition
+
+
+if __FILE__ == $0
+  license_text = 'Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.'
+  a = License_recognition.new.similarity(license_text, "./Package_license")
+  puts a
+end
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+