license_auto 0.1.1.2 → 0.1.1.3

data/remove/parser/manifest_parser.rb

@@ -1,66 +0,0 @@
-require 'yaml'
-
-require_relative '../../lib/db'
-require_relative '../api/pattern'
-
-class ManifestParser
-  def initialize(repo_filepath, repo_id)
-    @repo_filepath = repo_filepath
-    @manifest_file_list = []
-    ymls = api_get_repo_manifest_file_list(repo_id).values[0][0]
-    if ymls != nil
-      @manifest_file_list = ymls.gsub(' ', '').split(',')
-    end
-  end
-
-  def start
-    packs = []
-    $plog.debug("@manifest_file_list: #{@manifest_file_list}")
-    @manifest_file_list.each {|file_name|
-      manifest_file_namepath = "#{@repo_filepath}/#{file_name}"
-      $plog.debug("manifest_file_namepath: #{manifest_file_namepath}")
-      if File.exists?(manifest_file_namepath)
-        # TODO:
-        if file_name =~ API::FILE_NAME_PATTERN[:components_yml]
-        else
-          # normal 'manifest.yml' file
-          dependencies = ManifestYAML.new(manifest_file_namepath).get_dependencies
-          packs.concat(dependencies)
-        end
-
-      else
-        raise "Parameter error, @manifest_file_list: #{@manifest_file_list}"
-      end
-    }
-    packs
-  end
-end
-
-class ManifestYAML
-  attr_reader :contents, :pack_language, :manifest_file_namepath
-
-  def initialize(manifest_file_namepath)
-    @contents = YAML.load_file(manifest_file_namepath)
-    @pack_language = 'manifest.yml'
-  end
-
-  def get_dependencies()
-    if @contents['dependencies'] == nil
-      return []
-    else
-      return @contents['dependencies']
-    end
-  end
-
-  def get_exclude_files()
-    @contents['exclude_files']
-  end
-
-end
-
-if __FILE__ == $0
-  path = '/tmp/license_website/github.com/cloudfoundry/python-buildpack'
-  id = 81
-  p = ManifestParser.new(path, id)
-  p.start
-end

data/remove/parser/maven_parser.rb

@@ -1,91 +0,0 @@
-require 'open3'
-require_relative '../misc'
-
-class MavenParser
-
-  def initialize(repo_path)
-    @repo_path = repo_path
-    @package_list = Array.new
-    @pom_xml = /pom.xml/
-  end
-
-  def find_build_dot_pom
-    filenames = Misc::DirUtils.new(@repo_path).filter_filename(@pom_xml)
-
-    if filenames.size == 0
-      return false
-    else
-      return true
-    end
-  end
-
-  def start
-
-    if find_build_dot_pom == false
-      return []
-    end
-
-    cmd = 'mvn dependency:resolve'
-    flag = false
-    Dir.chdir(@repo_path)
-    Open3.popen3(cmd) {|i,o,e,t|
-      out = o.readlines
-      error = e.readlines
-      if error.length > 0
-        # TODO
-      elsif out.length > 0
-        out.each do |row|
-          flag = true if row == "[INFO] BUILD SUCCESS\n"
-        end
-      else
-        # TODO
-      end
-    }
-    if flag
-      cmd = 'mvn dependency:list'
-      flag = false
-      packages = []
-      Dir.chdir(@repo_path)
-      Open3.popen3(cmd) {|i,o,e,t|
-        out = o.readlines
-        error = e.readlines
-        if error.length > 0
-          # TODO
-        elsif out.length > 0
-          out.each do |row|
-            # p row
-            if row == "[INFO] The following files have been resolved:\n"
-              flag = true
-            elsif row == "[INFO] \n"
-              flag = false
-            elsif flag == true
-              packages << row
-            end
-          end
-        else
-          # TODO
-        end
-      }
-      packages.each do |line|
-        line.gsub!(/\[INFO\]/,'').strip!
-        pack = line.split("\:")
-        @package_list << {
-            :group   =>pack[0],
-            :name    =>pack[1],
-            :version => pack[3]
-        }
-      end
-    end
-
-    return @package_list
-  end
-
-end
-
-if __FILE__ == $0
-  path = "/home/li/ruby/maximus-master"
-  packs = MavenParser.new(path).start
-  packs.each do |row|
-    p row
-  end
-end

data/remove/parser/npm_parser.rb

@@ -1,82 +0,0 @@
-require 'json'
-require 'open3'
-require_relative '../../config/config'
-require_relative '../../lib/misc'
-require_relative '../api/npm_registry'
-require_relative '../api/pattern'
-
-class NpmParser
-  def initialize(repo_path)
-    @repo_path = repo_path
-    @package_json_pattern = /#{repo_path}\/.*package\.json$/
-  end
-
-  def start
-    pack_name_versions = []
-    filenames = Misc::DirUtils.new(@repo_path).filter_filename(@package_json_pattern)
-    filenames.each {|filename|
-      $plog.debug(filename)
-      j = JSON.parse(File.read(filename))
-      deps = j['dependencies']
-      dev_deps = j['devDependencies']
-      [deps, dev_deps].each {|d|
-        if d.nil?
-          next
-        end
-        d.each {|pack_name,semver|
-          if _is_valid_semver(semver)
-            certain_version = API::NpmRegistry.new(pack_name, semver).chose_one_available_version(semver)
-            pack = {'name' => pack_name, 'version' => certain_version}
-            pack_name_versions.push(pack)
-          elsif semver =~ API::SOURCE_URL_PATTERN[:npm_urls]
-            r = API::SOURCE_URL_PATTERN[:npm_urls].match(semver)
-            # TODO: save by original type
-            if r['host'] =~ API::SOURCE_URL_PATTERN[:github_dot_com]
-              source_url = "https://github.com/#{r['owner']}/#{r['repo']}"
-            else
-              source_url = semver
-            end
-            if r['ref'] == nil
-              version = 'master'
-            else
-              version = r['ref']
-            end
-            pack = {
-              'name' => pack_name,
-              'version' => version,
-              'uri' => source_url
-            }
-            pack_name_versions.push(pack)
-          else
-            pack = {
-              'name' => pack_name,
-              'version' => semver,
-              'uri' => nil 
-            } 
-            pack_name_versions.push(pack)
-            # raise "Unknown semver pattern: #{semver}, pack_name"
-          end
-        }
-      }
-    }
-    pack_name_versions
-  end
-
-  # DOC: https://github.com/npm/node-semver/blob/5f89ecbe78145ad0b501cf6279f602a23c89738d/test/index.js#L461
-  def _is_valid_semver(semver)
-    is_valid = false
-    cmd = "node -e \"var semver = require('semver'); var valid = semver.validRange('#{semver}'); console.log(valid)\""
-    $plog.debug(cmd)
-    Open3.popen3(cmd) {|i,o,e,t|
-      out = o.readlines
-      error = e.readlines
-      if error.length > 0
-        $plog.error(error)
-        raise "node -e evaluate script error: #{cmd}, #{error}"
-      elsif out.length > 0
-        is_valid = out[0].gsub(/\n/,'') != 'null'
-      end
-    }
-    is_valid
-  end
-end

data/remove/parser/pip_parser.rb

@@ -1 +0,0 @@
-# TODO:

data/remove/parser/rebar_parser.rb

@@ -1,61 +0,0 @@
-require 'json'
-require 'open3'
-require_relative '../../config/config'
-require_relative '../../lib/misc'
-require_relative '../api'
-
-class ErlangParser
-
-  def initialize(repo_path)
-    @repo_path = repo_path
-    @rebar_config_pattern = /#{repo_path}\/rebar\.config$/
-  end
-
-  def start
-    packs = []
-    filenames = Misc::DirUtils.new(@repo_path).filter_filename(@rebar_config_pattern)
-    if filenames.size == 1
-      cmd = "rebar list-deps"
-      Dir.chdir(@repo_path) {
-        Open3.popen3(cmd) {|i,o,e,t|
-          output = o.read
-          error = e.read
-          if error.length > 0
-            packs = filter_items(error)
-          elsif output.length > 0
-            packs = filter_items(output)
-          else
-            raise "#{self.class}.list_projects error error: #{cmd}, #{error}"
-          end
-        }
-      }
-    end
-    packs
-  end
-
-  def filter_items(output)
-    packs = []
-    output = output.gsub(/\s/, '').gsub(/ERROR:Missingdependencies:/, '').split(/\},\{/)
-    output.each {|d|
-      pattern = /dep,\"(?<local_dir>.+)\",(?<app_name>.+),\[*\],\{(?<repository_type>(git|hg|bzr)),\"(?<location_url>.*)\",\"(?<ref>.+)\"}/
-      matched = pattern.match(d)
-      if matched
-        # repository_type = matched[:repository_type]
-        source_url = matched[:location_url]
-        pack_version = matched[:ref]
-        if source_url =~ API::SOURCE_URL_PATTERN[:github]
-          pack_name = API::Github.new(source_url).repo
-          source_url = source_url.gsub(/git:\/\//, 'https://')
-        else
-          pack_name = source_url
-        end
-        packs.push({
-           'name' => pack_name,
-           'version' => pack_version,
-           'uri' => source_url
-        })
-      end
-    }
-    packs
-  end
-end

data/remove/recorder.rb

@@ -1,184 +0,0 @@
-# encoding: utf-8
-require 'csv'
-require 'json'
-require_relative '../config/config'
-require_relative './api'
-
-class PacksSaver
-  def initialize(repo_id, packs, lang, release_id)
-    @release_id = release_id
-    @repo_id = repo_id
-    @packs = packs
-    @lang = lang
-  end
-
-  def save()
-    if @lang == 'Golang'
-      save_golang
-    elsif @lang == 'Ruby'
-      save_ruby
-    elsif @lang == 'manifest.yml' or @lang == 'NodeJs' or @lang == 'Erlang'
-      save_manifest
-    elsif @lang == 'Gradle'
-      save_gradle
-    end
-  end
-
-  def save_gradle
-    @packs.each {|pack|
-      begin
-        $plog.debug(pack)
-        pack_name = [pack[:group], pack[:name]].join(':')
-        pack_version = pack[:version]
-        status = 10
-        homepage, source_url = nil, pack['uri']
-        license, cmt = nil, nil
-
-        pg_result = api_add_pack(pack_name, pack_version, @lang, homepage, source_url, license, status, cmt)
-        enqueue_result(pg_result)
-      rescue Exception => _
-        $plog.error(_)
-      end
-    }
-  end
-
-  def save_manifest
-    @packs.each {|pack|
-      begin
-        $plog.debug(pack)
-        pack_name, pack_version, status = pack['name'], pack['version'], 10
-        homepage, source_url = nil, pack['uri']
-        license, cmt = nil, nil
-
-        pg_result = api_add_pack(pack_name, pack_version, @lang, homepage, source_url, license, status, cmt)
-        enqueue_result(pg_result)
-      rescue Exception => _
-        $plog.error(_)
-      end
-    }
-  end
-
-  def save_ruby
-    def license_name_format(origin_license)
-      if origin_license.nil?
-        return nil
-      end
-      empty_pattern = /---\s\[\]\n/
-      license = origin_license.gsub(empty_pattern, '').gsub(/---\s\n[\.]+\n/, '').gsub(/---\n-\s/, '').gsub(/\n/, '').gsub(/['"\s]+$/, '').gsub(/^['"\s]+/, '')
-      if license.empty?
-        return nil
-      else
-        return license
-      end
-    end
-
-    @packs.each {|pack|
-      begin
-        pack[:license] = license_name_format(pack[:license])
-        # $plog.debug(pack)
-        pack_name, pack_version, status = pack[:pack_name], pack[:pack_version], pack[:status]
-        homepage, source_url = pack[:homepage], pack[:source_url]
-        # lang is a website in act
-        license, cmt, lang = pack[:license], pack[:cmt], pack[:language]
-
-        pg_result = api_add_pack(pack_name, pack_version, lang, homepage, source_url, license, status, cmt)
-        enqueue_result(pg_result)
-      rescue Exception => _
-        $plog.error(_)
-      end
-    }
-  end
-
-  # 处理api_add_pack()返回的结果
-  def enqueue_result(pg_result)
-    begin
-      if pg_result != nil
-        pack_id = pg_result['pack_id'].to_i
-        is_newbie = pg_result['is_newbie'] == 't'
-        api_add_product_repo_pack(@repo_id, pack_id, @release_id)
-
-        if is_newbie
-          queue_name = 'license_auto.pack'
-          $rmq.publish(queue_name, {:pack_id => pack_id}.to_json, check_exist=true)
-        end
-        $plog.debug("pack_id: #{pack_id}, is_newbie: #{is_newbie}")
-      else
-        $plog.error("#{pack_url} insert failed!")
-      end
-    rescue Exception => _
-      $plog.error(_)
-    end
-  end
-
-  def save_golang()
-    @packs.each { |pack_name, pack_url|
-      begin
-        $plog.debug("save_golange: #{pack_url}")
-
-        pack_version = 'unknown'
-        homepage = nil
-        source_url = nil
-        status = 10
-        cmt = nil
-
-        remote = API::RemoteSourceVCS.new(pack_url)
-
-        if remote.vcs == nil
-          status = 30
-          cmt = 'Unknown repo site'
-          pack_name = pack_name.split('/').last
-        else
-          pack_name = remote.vcs.repo
-          source_url = remote.vcs.repo_url
-          homepage = remote.get_homepage
-          last_commit = remote.vcs.last_commits
-          if last_commit
-            pack_version = last_commit['sha']
-          end
-        end
-        # $plog.debug("#{pack_version}, #{pack_url}")
-        license = nil
-        pg_result = api_add_pack(pack_name, pack_version, 'Golang', homepage, source_url, license, status, cmt)
-        enqueue_result(pg_result)
-      rescue Exception => _
-        $plog.error(_)
-      end
-    }
-  end
-
-end
-
-class PackUpdate
-  def initialize(pack_id, pack)
-    @pack_id = pack_id
-    @pack = pack
-  end
-
-  def update()
-    ok = api_update_pack_info(@pack_id, @pack)
-  end
-
-  def self.judge_pack_status(packer)
-    # TODO: @Micfan test it
-    if is_std_license(packer[:license]) and ( packer[:license_url] or packer[:license_text])
-      packer[:unclear_license] = nil
-      packer[:status] = 40
-    else
-      packer[:unclear_license] = packer[:license]
-      packer[:license] = nil
-      packer[:status] = 30
-    end
-    return packer
-  end
-
-  def self.is_std_license(license)
-    where = "where name = '#{license}'"
-    std_licenses = api_get_std_license_name(where)
-    return std_licenses.ntuples == 1
-  end
-
-end
-
-if __FILE__ == $0
-  p PackUpdate.is_std_license('MIT')
-end