license_auto 0.1.1.2 → 0.1.1.3

data/remove/api/mq.rb

@@ -1,30 +0,0 @@
-require 'bunny'
-
-module API
-
-class RabbitMQ
-  # "amqp://guest:guest@localhost:5672"
-  def initialize(rabbit_mq_conn_str)
-    @conn = Bunny.new(rabbit_mq_conn_str)
-    @conn.start
-    @ch = @conn.create_channel
-  end
-
-  def publish(queue_name, message, auto_release=true, durable=true, check_exist=false)
-    # TODO: @Micfan, 检查队列中是否存在该消息,如果已存在或正在处理,则返回false
-    def queue_exist(message)
-      false
-    end
-
-    q = @ch.queue(queue_name, :auto_release => auto_release, :durable => durable)
-    x = @ch.default_exchange
-
-    if check_exist and not queue_exist(message)
-      x.publish(message, :routing_key => q.name)
-    else
-      x.publish(message, :routing_key => q.name)
-    end
-  end
-end
-
-end

data/remove/api/npm_registry.rb

@@ -1,169 +0,0 @@
-require 'json'
-require 'open3'
-require 'httparty'
-
-require_relative '../../config/config'
-require_relative '../api/pattern'
-require_relative '../api/github'
-
-module API
-  # I have no time to research the npm cli source code(https://github.com/npm/npm-registry-client),
-  # quickly write this.
-  class NpmRegistry
-    def initialize(pack_name, pack_version)
-      @default_registry = 'http://registry.npmjs.org/'
-      @pack_name = pack_name
-      @pack_version = pack_version
-      @pack_meta = nil
-    end
-
-    # RESTful API: http://registry.npmjs.org/:pack_name
-    # TEST:        http://registry.npmjs.org/grunt
-    def get_package_meta
-      package_meta = nil
-      api_url = "#{@default_registry}#{@pack_name}"
-      $plog.debug(api_url)
-      response = HTTParty.get(api_url)
-      if response.code == 200
-        package_meta = JSON.parse(response.licenses)
-      elsif response.code == 404
-        $plog.error("!!! Npm registry API 404 Not found: #{response}")
-      else
-        $plog.error("!!! Npm registry API error response: #{response}")
-      end
-      package_meta
-    end
-
-    # RESTful API: http://registry.npmjs.org/grunt/?version=0.1.0
-    def get_package_info_by_version
-      package_info = nil
-      api_url = "#{@default_registry}#{@pack_name}/?version=#{@pack_version}"
-      $plog.debug(api_url)
-      response = HTTParty.get(api_url)
-      if response.code == 200
-        package_info = JSON.parse(response.licenses)
-      elsif response.code == 404
-        $plog.error("!!! Npm registry API 404 Not found: #{response}")
-      else
-        $plog.error("!!! Npm registry API error response: #{response}")
-      end
-      package_info
-    end
-
-    # DOC: https://www.npmjs.com/package/semver
-    # DOC: https://github.com/npm/node-semver
-    # sem_version_range: '~1.2.3'
-    def get_available_versions(sem_version_range)
-      available_versions = []
-      package_meta = get_package_meta
-      all_versions = package_meta['versions']
-      $plog.debug("sem_version_range: #{sem_version_range}")
-      all_versions.each_key {|version|
-        # node -e "var semver = require('semver'); var result = semver.satisfies('1.2.3', '1.x || >=2.5.0 || 5.0.0 - 7.2.3'); console.log(result);"
-        cmd = "node -e \"var semver = require('semver'); var available = semver.satisfies('#{version}', '#{sem_version_range}'); console.log(available);\""
-        # $plog.debug(cmd)
-        Open3.popen3(cmd) {|i,o,e,t|
-          out = o.readlines
-          error = e.readlines
-          if error.length > 0
-            $plog.error(error)
-            raise "node -e evaluate script error: #{cmd}, #{error}"
-          elsif out.length > 0
-            available = out[0].gsub(/\n/,'')
-            if available == 'true'
-              $plog.debug("available version: #{version}")
-              available_versions.push(version)
-            end
-          else
-            raise "node -e evaluate script error: #{cmd}, #{error}"
-          end
-        }
-      }
-      available_versions
-    end
-
-    def chose_one_available_version(sem_version_range)
-      available_versions = get_available_versions(sem_version_range)
-      # TODO: is it right?
-      $plog.debug("chosed version: #{available_versions.last}")
-      available_versions.last
-    end
-
-    def get_license_info()
-      license = nil
-      license_url = nil
-      license_text = nil
-      source_url = nil
-      homepage = nil
-
-      pack_info = get_package_info_by_version
-      $plog.debug("Npm registry pack_info: #{pack_info}")
-      if pack_info
-        licenses = pack_info['licenses']
-        if licenses != nil
-          # TODO: multi licenses:
-          # eg. http://registry.npmjs.org/grunt/?version=0.1.0
-          license = licenses[0]['type']
-
-          # TODO: convert http://github.com/cowboy/grunt/blob/master/LICENSE-MIT to raw
-          license_url = licenses[0]['url']
-          if license_url =~ API::SOURCE_URL_PATTERN[:github_html_page]
-            license_url, license_text = API::Github.convert_htmlpage_to_raw_url(license_url)
-          end
-        else
-          license = pack_info['license']
-        end
-
-        homepage = pack_info['homepage']
-        repository = pack_info['repository']
-        if repository
-          url = repository['url']
-          if repository['type'] == 'git'
-            source_url =
-                if url =~ /^git/
-                  url.gsub(/(git:\/\/|git\+ssh:\/\/git\@)/, 'http://').gsub(/\.git$/, '')
-                elsif url =~ /^http/
-                  url.gsub(/tree.*$/, '')
-                end
-          elsif repository['type'] =~ 'http'
-            source_url = repository['url'].gsub(/tree.*$/, '')
-          else
-            $plog.info("repository special type: #{repository}")
-          end
-        end
-
-        if license_text == nil
-          # TODO:
-        end
-      end
-      {
-        license: license,
-        license_url: license_url,
-        license_text: license_text,
-        source_url: source_url,
-        homepage: homepage
-      }
-    end
-  end
-end
-
-if __FILE__ == $0
-  # name = 'browserify'
-  # version = '12.0.1'
-  name = 'grunt'
-  version = '0.1.0'
-  n = API::NpmRegistry.new(name, version)
-  pack_info = n.get_package_info_by_version
-
-  license_info = n.get_license_info
-  $plog.info(license_info)
-
-  # pack_meta = n.get_package_meta
-  # p pack_meta
-  # p pack_meta['versions']
-
-  # sem_ver = '>=2.5.0'
-  # vs = n.get_available_versions(sem_ver)
-  # p vs
-end
-

data/remove/api/pattern.rb

@@ -1,33 +0,0 @@
-module API
-
-  SOURCE_URL_PATTERN = {
-    :github => /(?<protocol>(http[s]?|git))(:\/\/|@)(?<host>(www\.)?github\.com)(\/|:)(?<owner>.+)\/(?<repo>[^\/.]+)(?<ext>\.git)?/,
-    :git_kernel_org => /(?<protocol>http[s]?):\/\/(?<host>git\.kernel\.org)\/.+\/(?<owner>.+)\/(?<repo>[^\/.]+)(?<ext>\.git)?/,
-    # DOC: https://docs.npmjs.com/files/package.json#git-urls-as-dependencies
-    :npm_urls => /(?<protocol>git):\/\/(?<host>.*\..*)\/(?<owner>.+)\/(?<repo>[^\/#]+(\.git)?)(#(?<ref>.*))?/,
-    :bitbucket => /(?<protocol>http[s]?):\/\/(?<host>bitbucket\.org)\/(?<owner>.+)\/(?<repo>.+)(?<ext>\.git)?/,
-    :github_html_page => /(?<protocol>http[s]?):\/\/(?<host>(www\.)?github\.com)\/(?<owner>.+)\/(?<repo>[^\/]+)\/blob\/(?<branch>.+)\/(?<file_pathname>.+)?/,
-    :github_dot_com => /github\.com/,
-    :rubygems_org => /rubygems\.org/,
-    :google_source_com => /(?<protocol>http[s]?):\/\/(?<owner>.+)\.(?<host>googlesource\.com)\/(?<repo>[^\/.]+)(?<ext>\.git)?/,
-    :code_google_com => /(?<protocol>http[s]?):\/\/(?<host>code\.google\.com)\/(?<owner>p)\/(?<repo>[^\/.]+)?/,
-    :go_pkg_in => /(?<protocol>http[s]?):\/\/(?<host>gopkg\.in)\/((?<owner>[^\/.]+)\/(?<repo>[^\/.]+)\.(?<ref>.+)|(?<repo>[^\/.]+)\.(?<ref>[^\/.]+))/,
-    :golang_org => /(?<protocol>http[s]?):\/\/(?<host>.*golang\.org)\/(?<owner>[^\/.]+)\/(?<repo>[^\/.]+)/
-  }
-  FILE_NAME_PATTERN = {
-    :license_file => /(licen[sc]e|copying)+/i,
-    :readme_file => /readme/i,
-    :components_yml => /components\.yml/,
-    :notice_file => /(notice|copyright)/i,
-    :debian_copyright_file => /^[^\/]+\/debian\/copyright$/
-  }
-  FILE_TYPE_PATTERN = {
-    :tar_gz => /(tar\.gz|\.tgz)$/,
-    :tar_xz => /tar\.xz$/,
-    :tar_bz2 => /tar\.bz2$/
-  }
-  OS_PATTERN = {
-    :ubuntu => /^(?<distribution>(Ubuntu))-(?<distro_series>(Trusty))/i,
-    :centos => /^(?<distribution>(CentOS))-(?<distro_series>(7\.x))/i,
-  }
-end

data/remove/api/remote_source_package.rb

@@ -1,319 +0,0 @@
-require 'httparty'
-require 'anemone'
-require 'rubygems/package'
-require 'zlib'
-require 'xz'
-require 'open3'
-
-require_relative '../../config/config'
-require_relative '../../lib/api/helper'
-require_relative '../../lib/misc'
-
-module API
-
-  # TODO: replace this impl
-  # Ruby mixin's way
-  #
-  # module Person
-  #   def self.included x
-  #     ms = x.instance_methods(false)
-  #     [:xx, :yy, :zz].each do |xn|
-  #       raise "Please IMPL #{xn} Interface" unless ms.include?(xn)
-  #     end
-  #   end
-  # end
-  #
-  # class Customer
-  #   def xx;end
-  #   def yy;end
-  #   def zz;end
-  #   include Person
-  # end
-  #
-  # c = Customer.new
-  # c.xx
-  # c.yy
-  # c.zz
-  class RemoteSourcePackage
-    def initialize()
-    end
-
-    # OVERRIDE required
-    # return: homepage, download_url
-    def find_source_package_homepage_and_download_url()
-      raise 'Method must be overridden'
-    end
-
-    def download_source_code(source_code_url)
-      source_code_filename = source_code_url.split('/').last
-      source_code_path = "#{LAUNCHPAD_SOURCE_DIR}/#{source_code_filename}"
-      File.open(source_code_path, 'wb') do |f|
-        f.binmode
-        http_option = {
-          :timeout => HTTPARTY_DOWNLOAD_TIMEOUT
-        }
-        http_proxy = Misc.get_http_proxy
-        if http_proxy
-          http_option[:http_proxyaddr] = http_proxy[:addr]
-          http_option[:http_proxyport] = http_proxy[:port]
-        end
-        f.write(HTTParty.get(source_code_url, options=http_option).parsed_response)
-      end
-
-      return source_code_path
-    end
-
-    # Entry
-    def fetch_license_info_from_local_source()
-      license = nil
-      license_url = nil
-      license_text = nil
-      source_package_download_url = nil
-
-      # Attention, source package but binary package
-      source_package_homepage, source_package_download_url = find_source_package_homepage_and_download_url
-      $plog.debug("source_package_homepage: #{source_package_homepage}")
-      $plog.debug("source_package_download_url: #{source_package_download_url}")
-      if source_package_download_url
-        source_code_path = download_source_code(source_package_download_url)
-        $plog.debug("#{source_code_path}")
-        if source_code_path
-          reader = nil
-          if source_code_path =~ API::FILE_TYPE_PATTERN[:tar_gz]
-            reader = Zlib::GzipReader
-          elsif source_code_path =~ API::FILE_TYPE_PATTERN[:tar_xz]
-            reader = XZ::StreamReader
-          elsif source_code_path =~ API::FILE_TYPE_PATTERN[:tar_bz2]
-            # Bash script demo, MacOSX tar is not compatible
-            # $ tar --version
-            # tar (GNU tar) 1.27.1
-            # >>> Copyright (C) 2013 Free Software Foundation, Inc.
-            # >>> License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>.
-            # >>> This is free software: you are free to change and redistribute it.
-            # >> There is NO WARRANTY, to the extent permitted by law.
-            # >> Written by John Gilmore and Jay Fenlason.
-            # $ tar -tjvf bison_3.0.2.dfsg.orig.tar.bz2 | grep -i 'license\|copying\|readme' | awk '{ print $6 }'
-            # $ tar -xj --file=bison_3.0.2.dfsg.orig.tar.bz2 bison-3.0.2.dfsg/COPYING
-            # $ tar -xjO --file=bison_3.0.2.dfsg.orig.tar.bz2 bison-3.0.2.dfsg/COPYING
-            # $ tar -xjO --file=bison_3.0.2.dfsg.orig.tar.bz2 bison-3.0.2.dfsg/COPYING -C /dev/null
-            cmd_list_content = "tar -tjvf #{source_code_path} | grep -i 'license\\|copying' | awk '{ print $6 }'"
-            # MacOSX
-            # cmd_list_content = "tar -tjvf #{source_code_path} | grep -i 'license\\|copying' | awk '{ print $9 }'"
-            $plog.debug(cmd_list_content)
-            Open3.popen3(cmd_list_content) {|i,o,e,t|
-              out = o.readlines
-              error = e.readlines
-              if error.length > 0
-                # todo: move into exception.rb
-                raise "decompress error: #{source_code_path}, #{error}"
-              elsif out.length > 0
-                out.each {|line|
-                  license_file_path = line.gsub(/\n/, '')
-                  if @root_license_only and !API::Helper.is_root_file(license_file_path)
-                    next
-                  end
-                  cmd_read_content = "tar -xjO --file=#{source_code_path} #{license_file_path} -C /dev/null"
-                  Open3.popen3(cmd_read_content) {|i,o,e,t|
-                    out2 = o.read
-                    error = e.readlines
-                    if error.length > 0
-                      raise "cmd_read_content error: #{source_code_path}, #{license_file_path}, #{error}"
-                    elsif out2.length > 0
-                      license_text = out2
-                      license_url = license_file_path
-                      $plog.debug(license_text)
-                      break
-                    end
-                  }
-                }
-              end
-            }
-          else
-            $plog.error("source_package_download_url: #{source_package_download_url}, can NOT be uncompressed.")
-            return {}
-          end
-
-          if reader
-            tar_extract = Gem::Package::TarReader.new(reader.open(source_code_path))
-            tar_extract.rewind # The extract has to be rewinded after every iteration
-            tar_extract.each do |entry|
-              puts entry.full_name
-              # puts entry.directory?
-              # puts entry.file?
-              # puts entry.read
-              # Root dir files only
-              if entry.directory? or !API::Helper.is_root_file(entry.full_name)
-                # python-defaults-2.7.5/debian/copyright
-                #if API::Helper.is_debian_copyright_file(entry.full_name)
-                #  license_url = entry.full_name
-                #  license_text = entry.read
-                #  break
-                #else
-                  next
-               # end
-              end
-
-              if entry.file? and API::Helper.is_license_file(entry.full_name)
-                license_url = entry.full_name
-                license_text = entry.read
-
-                $plog.debug(entry.full_name)
-                $plog.debug(license_text)
-
-                # TODO: parser license info
-                break
-              end
-
-              # TODO:
-              # if entry.file? and API::Helper.is_readme_file(entry.full_name)
-              #   puts entry.full_name
-              #   # puts entry.read
-              #   # TODO: readme parser license info
-              #   break
-              # end
-
-            end
-            tar_extract.close ### to abstract out
-          end
-        end
-      end
-
-      if license_text
-        license = License_recognition.new.similarity(license_text, STD_LICENSE_DIR)
-      end
-      {
-        license: license,
-        license_url: license_url,
-        license_text: license_text,
-        source_url: source_package_download_url,
-        homepage: source_package_homepage
-      }
-    end
-
-  end
-
-  class Launchpad < RemoteSourcePackage
-    # DOC: https://launchpad.net/+apidoc/1.0.html
-    # The source code's license is no relation with which CPU architecture
-    def initialize(distribution, distro_series, binary_package_name, binary_package_version, architecture='amd64',
-                   root_license_only=true)
-      @site_url = 'https://launchpad.net'
-      @distribution = distribution
-      @distro_series = distro_series
-      @architecture = architecture
-      @binary_package_name = binary_package_name
-      @binary_package_version = binary_package_version
-      @download_dir = @download_dir
-
-      @source_url = nil
-      @source_path = nil
-      @root_license_only = root_license_only
-    end
-
-    def binary_package_link()
-      binary_package_url = "#{@site_url}/#{@distribution}/#{@distro_series}/#{@architecture}/#{@binary_package_name}/#{@binary_package_version}"
-    end
-
-    # @OVERRIDED
-    def find_source_package_homepage_and_download_url()
-      homepage = _find_source_package_homepage
-      download_url = nil
-      if homepage
-        download_url = _find_source_code_download_url(homepage)
-      end
-      return homepage, download_url
-    end
-
-    def _find_source_package_homepage()
-      source_package_homepage = nil
-      opts = {:discard_page_bodies => true, :depth_limit => 0}
-
-      $plog.debug("binary_package_link: #{binary_package_link}")
-
-      Anemone.crawl(binary_package_link, opts) do |anemone|
-        anemone.on_every_page do |page|
-          xpath = "//dd[@id='source']/a[1]"
-          page.doc.xpath(xpath).each {|text|
-            abs_href = text.css('/@href')
-            if abs_href
-              source_package_homepage = "#{@site_url}#{abs_href}"
-              break
-            end
-          }
-        end
-      end
-      return source_package_homepage
-    end
-
-    def _find_source_code_download_url(source_package_homepage)
-      source_code_download_url = nil
-
-      opts = {:discard_page_bodies => true, :depth_limit => 0}
-      Anemone.crawl(source_package_homepage, opts) do |anemone|
-        anemone.on_every_page do |page|
-          xpath = "//div[@id='source-files']/table/tbody/tr/td/a[contains(@href, '.orig.')]"
-          target_links = page.doc.xpath(xpath)
-          if target_links.size == 0
-            # Eg. https://launchpad.net/ubuntu/+source/wireless-crda/1.16
-            xpath = "//div[@id='source-files']/table/tbody/tr/td/a[not(contains(@href, '.dsc'))]"
-            target_links = page.doc.xpath(xpath)
-          end
-          # puts target_links
-          target_links.each {|text|
-              full_href = text.attr('href')
-              if full_href
-                source_code_download_url = full_href
-                break
-              end
-          }
-        end
-      end
-      return source_code_download_url
-    end
-
-    # def make_up_license_url()
-    #   license_url = 'http://bazaar.launchpad.net/~ubuntu-branches/ubuntu/vivid/anacron/vivid/view/head:/COPYING'
-    # end
-
-  end
-
-  class ManifestPackage < RemoteSourcePackage
-    def initialize(source_code_download_url, root_license_only=true)
-      @download_url = source_code_download_url
-      @root_license_only = root_license_only
-    end
-
-    def find_source_package_homepage_and_download_url
-      homepage = nil
-      return homepage, @download_url
-    end
-  end
-end
-
-if __FILE__ == $0
-  distribution = 'ubuntu'
-  distro_series = 'trusty'
-  name = 'anacron'
-  version = '2.3-20ubuntu1'
-
-  name = 'sg3-utils'
-  version = '1.36-1ubuntu1'
-
-  name = 'wireless-crda'
-  version = '1.16'
-
-  # .bz2
-  name = 'libpython-stdlib'
-  version = '2.7.5-5ubuntu3'
-  a = API::Launchpad.new(distribution, distro_series, name, version)
-  # a = API::ManifestPackage.new('https://pivotal-buildpacks.s3.amazonaws.com/python/binaries/cflinuxfs2/libmemcache.tar.gz')
-  license_info = a.fetch_license_info_from_local_source
-  p license_info
-
-
-# ii  anacron                             2.3-20ubuntu1                    amd64        cron-like program that doesn't go by time
-# ii  apparmor                            2.8.95~2430-0ubuntu5.3           amd64        User-space parser utility for AppArmor
-# ii  apparmor-utils                      2.8.95~2430-0ubuntu5.3           amd64        Utilities for controlling AppArmor
-# ii  apt                                 1.0.1ubuntu2.10                  amd64        commandline package manager
-# ii  apt-utils                           1.0.1ubuntu2.10                  amd64        package management related utility programs
-end