license_auto 0.1.1.1

data/lib/recorder.rb

@@ -0,0 +1,184 @@
+# encoding: utf-8
+require 'csv'
+require 'json'
+require_relative '../config/config'
+require_relative './api'
+
+class PacksSaver
+  def initialize(repo_id, packs, lang, release_id)
+    @release_id = release_id
+    @repo_id = repo_id
+    @packs = packs
+    @lang = lang
+  end
+
+  def save()
+    if @lang == 'Golang'
+      save_golang
+    elsif @lang == 'Ruby'
+      save_ruby
+    elsif @lang == 'manifest.yml' or @lang == 'NodeJs' or @lang == 'Erlang'
+      save_manifest
+    elsif @lang == 'Gradle'
+      save_gradle
+    end
+  end
+
+  def save_gradle
+    @packs.each {|pack|
+      begin
+        $plog.debug(pack)
+        pack_name = [pack[:group], pack[:name]].join(':')
+        pack_version = pack[:version]
+        status = 10
+        homepage, source_url = nil, pack['uri']
+        license, cmt = nil, nil
+
+        pg_result = api_add_pack(pack_name, pack_version, @lang, homepage, source_url, license, status, cmt)
+        enqueue_result(pg_result)
+      rescue Exception => _
+        $plog.error(_)
+      end
+    }
+  end
+
+  def save_manifest
+    @packs.each {|pack|
+      begin
+        $plog.debug(pack)
+        pack_name, pack_version, status = pack['name'], pack['version'], 10
+        homepage, source_url = nil, pack['uri']
+        license, cmt = nil, nil
+
+        pg_result = api_add_pack(pack_name, pack_version, @lang, homepage, source_url, license, status, cmt)
+        enqueue_result(pg_result)
+      rescue Exception => _
+        $plog.error(_)
+      end
+    }
+  end
+
+  def save_ruby
+    def license_name_format(origin_license)
+      if origin_license.nil?
+        return nil
+      end
+      empty_pattern = /---\s\[\]\n/
+      license = origin_license.gsub(empty_pattern, '').gsub(/---\s\n[\.]+\n/, '').gsub(/---\n-\s/, '').gsub(/\n/, '').gsub(/['"\s]+$/, '').gsub(/^['"\s]+/, '')
+      if license.empty?
+        return nil
+      else
+        return license
+      end
+    end
+
+    @packs.each {|pack|
+      begin
+        pack[:license] = license_name_format(pack[:license])
+        # $plog.debug(pack)
+        pack_name, pack_version, status = pack[:pack_name], pack[:pack_version], pack[:status]
+        homepage, source_url = pack[:homepage], pack[:source_url]
+        # lang is a website in act
+        license, cmt, lang = pack[:license], pack[:cmt], pack[:language]
+
+        pg_result = api_add_pack(pack_name, pack_version, lang, homepage, source_url, license, status, cmt)
+        enqueue_result(pg_result)
+      rescue Exception => _
+        $plog.error(_)
+      end
+    }
+  end
+
+  # 处理api_add_pack()返回的结果
+  def enqueue_result(pg_result)
+    begin
+      if pg_result != nil
+        pack_id = pg_result['pack_id'].to_i
+        is_newbie = pg_result['is_newbie'] == 't'
+        api_add_product_repo_pack(@repo_id, pack_id, @release_id)
+
+        if is_newbie
+          queue_name = 'license_auto.pack'
+          $rmq.publish(queue_name, {:pack_id => pack_id}.to_json, check_exist=true)
+        end
+        $plog.debug("pack_id: #{pack_id}, is_newbie: #{is_newbie}")
+      else
+        $plog.error("#{pack_url} insert failed!")
+      end
+    rescue Exception => _
+      $plog.error(_)
+    end
+  end
+
+  def save_golang()
+    @packs.each { |pack_name, pack_url|
+      begin
+        $plog.debug("save_golange: #{pack_url}")
+
+        pack_version = 'unknown'
+        homepage = nil
+        source_url = nil
+        status = 10
+        cmt = nil
+
+        remote = API::RemoteSourceVCS.new(pack_url)
+
+        if remote.vcs == nil
+          status = 30
+          cmt = 'Unknown repo site'
+          pack_name = pack_name.split('/').last
+        else
+          pack_name = remote.vcs.repo
+          source_url = remote.vcs.repo_url
+          homepage = remote.get_homepage
+          last_commit = remote.vcs.last_commits
+          if last_commit
+            pack_version = last_commit['sha']
+          end
+        end
+        # $plog.debug("#{pack_version}, #{pack_url}")
+        license = nil
+        pg_result = api_add_pack(pack_name, pack_version, 'Golang', homepage, source_url, license, status, cmt)
+        enqueue_result(pg_result)
+      rescue Exception => _
+        $plog.error(_)
+      end
+    }
+  end
+
+end
+
+class PackUpdate
+  def initialize(pack_id, pack)
+    @pack_id = pack_id
+    @pack = pack
+  end
+
+  def update()
+    ok = api_update_pack_info(@pack_id, @pack)
+  end
+
+  def self.judge_pack_status(packer)
+    # TODO: @Micfan test it
+    if is_std_license(packer[:license]) and ( packer[:license_url] or packer[:license_text])
+      packer[:unclear_license] = nil
+      packer[:status] = 40
+    else
+      packer[:unclear_license] = packer[:license]
+      packer[:license] = nil
+      packer[:status] = 30
+    end
+    return packer
+  end
+
+  def self.is_std_license(license)
+    where = "where name = '#{license}'"
+    std_licenses = api_get_std_license_name(where)
+    return std_licenses.ntuples == 1
+  end
+
+end
+
+if __FILE__ == $0
+  p PackUpdate.is_std_license('MIT')
+end

data/lib/script/apt-get.deps.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+cat /root/.bash_profile
+export license_auto_rabbit_mq="amqp://guest:guest@localhost:5672"
+export github_username=Your Github Username
+# https://help.github.com/articles/creating-an-access-token-for-command-line-use/
+export github_password=Your Github Personal access tokens
+export license_auto_proxy='proxy.foo.com:8080'
+
+apt-get install -y rabbitmq-server \
+    libpq-dev build-essential \
+    libxml2 \
+    libxslt bzip2 cmake golang \
+    rebar gradle maven
+
+# If mac osx
+# brew install icu4c
+
+rabbitmq-plugins enable rabbitmq_management
+rabbitmqctl stop # 停止Erlang节点
+service rabbitmq-server start # 启动Erlang节点
+
+bundle install
+
+pip install -r ../requirements.txt
+cp conf/supervisord.conf /etc/supervisord.conf
+supervisord
+
+cp conf/.git-credentials ~/.git-credentials
+git config --global credential.helper 'store --file ~/.git-credentials'
+git config --global http.proxy http://proxy.mycompany:80
+
+npm install -g semver
+
+
+
+

data/lib/script/debian_dpkg_list.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+dpkg -l | grep -e "^ii.*" | awk '{ print $2","$3","$4 }' > dpkg_list.txt

data/lib/script/rubygems.org.importdb.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+
+su postgres
+cd ~
+# TODO: write a Ruby spider to auto
+wget https://s3-us-west-2.amazonaws.com/rubygems-dumps/production/public_postgresql/2015.09.07.21.21.02/public_postgresql.tar
+
+tar -xvf public_postgresql.tar
+gunzip public_postgresql/databases/PostgreSQL.sql.gz
+
+rm -rf public_postgresql*
+
+sql_filename=public_postgresql/databases/PostgreSQL.sql
+
+dbname=gemData
+dropdb $dbname
+createdb $dbname
+psql $dbname -c 'create extension hstore;'
+
+psql $dbname < $sql_filename
+echo 'OK'

data/license_auto.gemspec

@@ -0,0 +1,44 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'license_auto/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "license_auto"
+  spec.version       = LicenseAuto::VERSION::STRING
+  spec.authors       = ['MineWorks']
+  spec.email         = [""]
+
+  spec.summary       = %q{License Automation Toolkit}
+  spec.description   = %q{LicenseAuto is a library for Open Source License collection job. Supported Language Package
+                          Management including:
+                            Ruby Gems, Java Maven Gradle, NodeJS npm, Erlang rebar, Debian launchpad.net, Golang}
+
+  spec.homepage      = "https://github.com/mineworks/license_auto"
+  spec.license       = "MIT"
+
+  spec.metadata = {
+      'repository_url' => 'https://github.com/mineworks/license_auto.git'
+  }
+
+  # Prevent pushing this gem to RubyGems.org by setting 'allowed_push_host', or
+  # delete this section to allow pushing this gem to any host.
+  if spec.respond_to?(:metadata)
+    spec.metadata['allowed_push_host'] = "https://rubygems.org"
+  else
+    raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
+  end
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "bin"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "hashie"
+  spec.add_runtime_dependency "json"
+  spec.add_runtime_dependency "gems"
+  spec.add_runtime_dependency "github_api"
+
+  spec.add_development_dependency "bundler", "~> 1.10"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

data/remove/license_auto

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+puts "Hello, license_auto!"

data/remove/mq_pack.rb

@@ -0,0 +1,219 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+require File.expand_path('../config/environment', __FILE__)
+
+if ENV['RACK_ENV'] == 'development'
+end
+
+
+
+require_relative '../lib/cloner'
+require_relative '../lib/db'
+require_relative '../lib/recorder'
+require_relative '../lib/api'
+require_relative '../config/config'
+require_relative '../lib/api'
+
+def fetch_license_info_by_source(packer, status=nil)
+    # TODO: @Micfan, simple it
+    source_url = packer[:source_url]
+    homepage = packer[:homepage]
+    github_pattern = API::SOURCE_URL_PATTERN[:github]
+    if source_url =~ github_pattern
+      github_url = source_url
+    elsif homepage =~ github_pattern
+      github_url = homepage
+    else
+      github_url = nil
+    end
+
+    # $plog.debug("github_url: #{github_url}")
+
+    bitbucket_url = nil
+    bitbucket_pattern = /http[s]?:\/\/bitbucket\.org/
+    if source_url =~ bitbucket_pattern
+      bitbucket_url = source_url
+    elsif homepage =~ bitbucket_pattern
+      bitbucket_url = homepage
+    else
+      bitbucket_url = nil
+    end
+
+    license_info = {:license => nil, :license_text => nil, :license_url => nil}
+    if github_url != nil
+      packer[:source_url] = github_url
+
+      # 1. find a version tag, if version tag not exists, go to default branch
+      # 2. if license file not found in a version tag, go to default branch
+      extractor = API::Github.new(github_url, db_ref=packer[:version])
+      license_info = extractor.get_license_info
+      if license_info[:license_url] == nil
+        default_branch, switched = extractor.switch_to_default_branch
+        if switched
+          license_info = extractor.get_license_info
+        end
+      end
+
+      packer = packer.merge(license_info)
+    elsif bitbucket_url != nil
+      packer[:source_url] = bitbucket_url
+      extractor = API::Bitbucket.new(bitbucket_url)
+      license_info = extractor.get_license_info
+      packer = packer.merge(license_info)
+    end
+
+    if status
+      packer[:status] = status
+    else
+      packer = PackUpdate.judge_pack_status(packer)
+    end
+
+  return packer
+end
+
+def worker(body)
+  begin
+    pack_item = JSON.parse(body)
+    pack_id = pack_item['pack_id']
+    pack = api_get_pack_by_id(pack_id)
+    if pack.nil?
+      status = 20
+      cmt = "Package not found, pack_id: #{pack_id}"
+      api_setup_pack_status(pack_id, status, cmt)
+      $plog.fatal(cmt)
+      return
+    else
+      _status = pack['status'].to_i
+      if _status >= 30
+        info = "This package(pack_id=#{pack_id}, status=#{_status}) have no permission to self run"
+        $plog.info(info)
+        return
+      end
+    end
+
+    $plog.debug("pack: #{pack}")
+
+    lang = pack['lang']
+
+    packer = {
+      :version => pack['version'],
+      :source_url => pack['source_url'],
+      :license_url => pack['license_url'],
+      :homepage => pack['homepage'],
+      :license => pack['license'],
+      :license_text => pack['license_text'],
+      # TODO： this status should rethink
+      :status => 30
+    }
+    packer = fetch_license_info_by_source(packer)
+    if packer[:status] < 40 and packer[:source_url] == nil
+      if lang =~ API::OS_PATTERN[:ubuntu]
+        ubuntu_lang_pattern = API::OS_PATTERN[:ubuntu]
+        lang_group = ubuntu_lang_pattern.match(lang)
+        launchpad = API::Launchpad.new(lang_group[:distribution].downcase, lang_group[:distro_series].downcase,
+                                       pack['name'].split(':').first, pack['version'])
+        license_info = launchpad.fetch_license_info_from_local_source
+        packer = packer.merge(license_info)
+        packer = PackUpdate.judge_pack_status(packer)
+      elsif lang =~ API::OS_PATTERN[:centos]
+          # *.rpm, $yum
+      elsif lang == 'Gradle'
+        gropu_id, name_id = pack['name'].split(':')
+        m = API::MavenCentralRepository.new(gropu_id, name_id, pack['version'])
+        license_info = m.get_license_info
+        packer = packer.merge(license_info)
+        if license_info[:licenses].size > 0
+          # TODO: multi license
+          packer = packer.merge(license_info[:licenses][0])
+        end
+        packer = PackUpdate.judge_pack_status(packer)
+      elsif lang == 'NodeJs'
+        registry = API::NpmRegistry.new(pack['name'], pack['version'])
+        license_info = registry.get_license_info
+        packer = packer.merge(license_info)
+        packer = PackUpdate.judge_pack_status(packer)
+        if packer[:status] < 40 and packer[:status] >= 30
+          source_license_info = fetch_license_info_by_source(packer)
+          packer = packer.merge(source_license_info)
+          if packer[:license].nil?
+            packer[:license] = license_info[:license]
+          end
+          packer = PackUpdate.judge_pack_status(packer)
+        end
+      end
+    elsif lang == 'manifest.yml'
+      source_code_download_url = api_get_manifest_download_url(pack_id)[0]['source_url']
+      if source_code_download_url == nil
+        raise "pack_id(#{pack_id}) source_url is null"
+      else
+        manifest = API::ManifestPackage.new(source_code_download_url)
+        license_info = manifest.fetch_license_info_from_local_source
+        packer = packer.merge(license_info)
+        packer = PackUpdate.judge_pack_status(packer)
+      end
+    else
+      $plog.info("packer: #{packer}")
+      # TODO: def is_github_source_url
+      if (packer[:source_url] =~ API::SOURCE_URL_PATTERN[:github]).nil? and packer[:homepage] != nil
+        $plog.info("Homepage Spider working: #{packer[:homepage]}")
+        spider_source_url = API::Spider.new(packer[:homepage], pack['name']).find_source_url
+        $plog.debug("spider_source_url: #{spider_source_url}")
+        if spider_source_url
+          packer[:source_url] = spider_source_url
+          packer = fetch_license_info_by_source(packer, status=31)
+        end
+      end
+      $plog.error("!!! Unresolved pack: #{pack}")
+    end
+
+    updater = PackUpdate.new(pack_id, packer)
+    flag = updater.update()
+    if flag
+      $plog.info("update success: pack_id: #{pack_id}, packer: #{packer}")
+    else
+      $plog.info("!!! update failed: pack_id: #{pack_id}, packer: #{packer}")
+    end
+  rescue Git::GitExecuteError => e
+    $plog.fatal(e)
+    api_setup_pack_status(pack_id, 21, e.to_s)
+  rescue OpenSSL::SSL::SSLError => e
+    $plog.fatal(e)
+    api_setup_pack_status(pack_id, 22, e.to_s)
+  rescue Exception => e
+    api_setup_pack_status(pack_id, 20, e.to_s)
+    $plog.fatal(e)
+  end
+  
+end
+
+def main()
+  conn = Bunny.new
+  conn.start
+
+  ch   = conn.create_channel
+  q    = ch.queue("license_auto.pack", :durable => true)
+
+  ch.prefetch(1)
+  puts " [*] MQ-pack: Waiting for messages. To exit press CTRL+C"
+
+  begin
+    q.subscribe(:manual_ack => true, :block => true) do |delivery_info, properties, body|
+      puts " [x] MQ-pack:  Received '#{body}'"
+
+      worker(body)
+      sleep 1.0
+      puts " [x] MQ-pack: Done"
+      ch.ack(delivery_info.delivery_tag)
+    end
+  rescue Interrupt => _
+    conn.close
+  end
+end
+
+if __FILE__ == $0
+  # pack_id = 1796
+  # body = '{"pack_id":' + "#{pack_id}}"
+  # worker(body)
+  main
+end