license_auto 0.1.1.1

data/lib/api/google_source_com.rb

@@ -0,0 +1,68 @@
+require 'anemone'
+
+require_relative '../api/pattern'
+require_relative '../../lib/misc'
+
+module API
+  class GoogleSourceCom
+    attr_reader :repo_url, :protocol, :host, :owner, :repo, :ref
+
+    # repo_url: https://go.googlesource.com/crypto
+    def initialize(repo_url, db_ref=nil)
+      @repo_url = repo_url
+
+      repo_url_pattern = API::SOURCE_URL_PATTERN[:google_source_com]
+      regex_group = repo_url_pattern.match(repo_url)
+      @protocol = regex_group[:protocol]
+      @host = regex_group[:host]
+      @owner = regex_group[:owner]
+      @repo = regex_group[:repo]
+
+      @http_option = {}
+      http_proxy = Misc.get_http_proxy
+      if http_proxy
+        @http_option[:proxy_host] = http_proxy[:addr]
+        @http_option[:proxy_port] = http_proxy[:port]
+      end
+      @ref = db_ref
+
+    end
+
+    # URL.
+    def last_commits
+      last_commit = nil
+      opts = {:discard_page_bodies => true, :depth_limit => 0}.merge(@http_option)
+      commit_page = "#{@repo_url}"
+      Anemone.crawl(commit_page, opts) do |anemone|
+        anemone.on_every_page do |page|
+          xpath = "//ol[@class='CommitLog']/li[1]/a[1]"
+          target_link = page.doc.xpath(xpath)
+          # p target_link
+          if target_link.size == 0
+            raise "last_commit error: #{self}, #{@repo_url}"
+          else
+            # full_href = text.attr('href')
+            short_sha = target_link.text()
+            full_sha = target_link.attr('href').value
+            last_commit = {
+              'sha' => full_sha.split('/+/').last
+            }
+          end
+        end
+      end
+      last_commit
+    end
+  end
+end
+
+
+if __FILE__ == $0
+  url = 'https://go.googlesource.com/crypto'
+  g = API::GoogleSourceCom.new(url)
+   p g.last_commits
+#   p g.protocol
+#   p g.host
+#   p g.owner
+#   p g.repo
+#   p g.repo_url
+end

data/lib/api/gradle2.rb

@@ -0,0 +1,41 @@
+module API
+  class Gradle
+    def initialize(gradle_file)
+      @gradle_file = gradle_file
+      @websites = {
+        mvn: 'http://mvnrepository.com/',
+        maven: 'https://maven-repository.com/search',
+        rpmfind: 'http://www.rpmfind.net/'
+      }
+
+    end
+
+    # DOC: http://pkaq.github.io/gradledoc/docs/userguide/ch11/tutorial_gradle_command_line.html#sec:listing_dependencies
+    def list_dependencies()
+      deps = {
+        :pack_name_1 => 'pack_version_1',
+        :pack_name_2 => 'pack_version_2'
+      }
+    end
+
+    # JSON: http://search.maven.org/solrsearch/select?q=g:%22org.apache.commons%22&rows=20&wt=json
+    # JSON: http://search.maven.org/solrsearch/select?q=g:%22org.apache.commons%22%20AND%20a:%22commons-lang3%22&rows=2000&wt=json
+    # JSON: http://search.maven.org/solrsearch/select?q=g:%22org.apache.commons%22%20AND%20a:%22commons-lang3%22%20AND%20v:%223.0%22&rows=2000&wt=json
+    # http://search.maven.org/solrsearch/select?q=g:%22org.apache.maven.indexer%22+AND+a:%22maven-indexer%22&rows=20&core=gav
+
+    def fetch_license_info_from_website()
+      license = nil
+      license_url = nil
+      license_text = nil
+      source_code_download_url = nil
+      source_package_page_link = nil
+      license_info = {
+        license: license,
+        license_url: license_url,
+        license_text: license_text,
+        source_url: source_code_download_url,
+        homepage: source_package_page_link
+      }
+    end
+  end
+end

data/lib/api/helper.rb

@@ -0,0 +1,26 @@
+require_relative './pattern'
+module API
+
+  class Helper
+    def self.is_license_file(filename)
+      return filename =~ API::FILE_NAME_PATTERN[:license_file]
+    end
+
+    # file_pathname = 'foo/to/bar'
+    def self.is_root_file(file_pathname)
+      return file_pathname.split('/').size == 2
+    end
+
+    def self.is_readme_file(filename)
+      return filename =~ API::FILE_NAME_PATTERN[:readme_file]
+    end
+
+    def self.is_notice_file(filename)
+      return filename =~ API::FILE_NAME_PATTERN[:notice_file]
+    end
+    def self.is_debian_copyright_file(file_pathname)
+      return file_pathname =~ API::FILE_NAME_PATTERN[:debian_copyright_file]
+    end
+  end
+
+end

data/lib/api/j_center.rb

@@ -0,0 +1,23 @@
+module API
+  class JCenter
+
+    # Site: https://bintray.com/bintray/jcenter
+    # DOC: https://bintray.com/docs/api/
+    def initialize(group, name, version)
+    end
+
+    def get_license_info()
+      {
+        license: nil,
+        license_text: nil,
+        source_url: nil,
+        homepage: nil,
+
+      }
+    end
+
+
+  end
+end
+
+

data/lib/api/maven_central_repository.rb

@@ -0,0 +1,192 @@
+require 'json'
+require 'nokogiri'
+require 'httparty'
+require_relative '../../config/config'
+
+module API
+  # DOC: http://search.maven.org/#api
+  class MavenCentralRepository
+    def initialize(group_id, artifact_id, version)
+      @group_id = group_id
+      @artifact_id = artifact_id
+      @version = version
+      # @scope = scope
+      # TODO: what is this field?
+      @classifier = ''
+
+      @api_url = "http://search.maven.org/solrsearch/select?q=g%3A%22com.google.inject%22&rows=20&wt=json"
+    end
+
+    # DOC: http://search.maven.org/solrsearch/select?q=g:"com.google.inject"+AND+a:"guice"&core=gav&rows=20&wt=json
+    def select()
+      url = "http://search.maven.org/solrsearch/select?q=g:\"#{@group_id}\"+AND+a:\"#{@artifact_id}\"&core=gav&rows=20&wt=json"
+      url = URI.escape(url)
+      $plog.debug(url)
+      response = HTTParty.get(url)
+      if response.code == 200
+        query_set = JSON.parse(response.licenses)
+      else
+        raise "CentralRepository select error: #{response}"
+      end
+    end
+
+    # Eg: http://search.maven.org/solrsearch/select?q=g:%22com.google.inject%22%20AND%20a:%22guice%22%20AND%20v:%223.0%22%20AND%20l:%22javadoc%22%20AND%20p:%22jar%22&rows=20&wt=json
+    def advance_search
+      url = "http://search.maven.org/solrsearch/select?q=g:\"#{@group_id}\" AND a:\"#{@artifact_id}\" AND v:\"#{@version}\" AND l:\"#{@classifier}\" AND p:\"jar\"&rows=20&wt=json"
+      url = URI.escape(url)
+      $plog.debug("api_url: #{url}")
+      response = HTTParty.get(url)
+      if response.code == 200
+        query_set = JSON.parse(response.licenses)
+      else
+        raise "CentralRepository select error: #{response}"
+      end
+    end
+
+    # POM: https://repo1.maven.org/maven2/com/google/inject/guice/3.0/guice-3.0.pom
+    def get_package_pom(group, name, version)
+      pom = nil
+
+      central_prefix = 'https://repo1.maven.org/maven2'
+      central_body = [group.gsub(/\./, '/'), name, version].join('/')
+      central_tail = "#{name}-#{version}.pom"
+      pom_url = [central_prefix, central_body, central_tail].join('/')
+      # http://stackoverflow.com/questions/25814210/opensslsslsslerror-ssl-connect-syscall-returned-5-errno-0-state-sslv3-read
+      opts = {
+        :ssl_version => 'TLSv1'
+      }
+      response = HTTParty.get(pom_url, options=opts)
+      if response.code == 200
+        $plog.debug("pom_url: #{pom_url}")
+        pom = response.licenses
+      else
+        $plog.error("CentralRepository get_package_pom error: pom_url: #{pom_url}, #{response}")
+      end
+      return pom_url, pom
+    end
+
+    def get_license_info()
+      license_info = {
+        homepage: nil,
+        source_url: nil,
+        licenses: [],
+        project_url: nil,
+        pom_url: nil
+      }
+      query_set = select
+
+      if query_set
+        query_set['response']['docs'].each {|d|
+          version = d['v']
+          ec = d['ec']
+          if version == @version && ec.index('.pom')
+            license_info[:project_url] = "https://maven-repository.com/artifact/#{@group_id}/#{@artifact_id}/#{@version}"
+            pom_url, pom = get_package_pom(@group_id, @artifact_id, @version)
+            if pom
+              license_info[:pom_url] = pom_url
+
+              # $plog.debug("pom: #{pom}")
+              doc = Nokogiri::XML(pom).remove_namespaces!
+
+              source_code_node = doc.xpath("/project/scm/url")
+              license_info[:source_url] = source_code_node.text if source_code_node
+
+              homepage_node = doc.xpath("/project/url")
+              license_info[:homepage] = homepage_node.text if homepage_node
+
+              xpath = "//licenses/license"
+              licenses = doc.xpath(xpath)
+              $plog.debug("licenses: #{licenses.to_xml}")
+
+              # Multi licenses: https://maven-repository.com/artifact/org.cryptacular/cryptacular/1.0
+              licenses.each {|node|
+                license = nil
+                license_url = nil
+                license_text = nil
+                if node.xpath(".//name")
+                  license = node.xpath(".//name").text
+                  # $plog.debug("license: #{license}")
+                end
+                license_url = node.xpath(".//url").text if licenses.xpath(".//url")
+
+                # TODO: find a license_text demo
+                # license_text = licenses.xpath(".//text").text if licenses.xpath(".//text")
+
+                license_info[:licenses] << {
+                  license: license,
+                  license_url: license_url,
+                  license_text: license_text
+                }
+              }
+              if licenses.size == 0
+                $plog.debug("licenses.size: #{licenses.size}")
+                # Comment license text info: eg. https://repo1.maven.org/maven2/commons-io/commons-io/2.4/commons-io-2.4.pom
+                comment_node = doc.xpath("/comment()[contains(., 'license')]")
+                if comment_node.size > 0
+                  license_text = comment_node.to_xml
+                  license_info[:licenses] << {
+                    license: 'UNKNOWN',
+                    license_url: nil,
+                    license_text: license_text
+                  }
+                end
+              end
+            end
+
+            break
+          end
+        }
+      end
+      license_info
+    end
+  end
+end
+
+if __FILE__ == $0
+  # One license
+  # item = {:group=>"net.sourceforge.nekohtml", :name=>"nekohtml", :version=>"1.9.20"}
+
+  # Two license
+  # item = {:group=>"org.cryptacular", :name=>"cryptacular", :version=>"1.0"}
+
+  # License in comments
+  item = {:group=>"commons-io", :name=>"commons-io", :version=>"2.4"}
+
+
+  'aopalliance:aopalliance:1.0'
+  'cglib:cglib-nodep:3.1'
+  'com.beust:jcommander:1.48'
+  'com.google.guava:guava:18.0'
+  'com.google.inject:guice:no_aop:4.0'
+  'com.jayway.awaitility:awaitility:1.6.3'
+  'javax.inject:javax.inject:1'
+  'junit:junit:4.10'
+  'org.apache.ant:ant:1.7.0'
+  'org.apache.ant:ant-launcher:1.7.0'
+  'org.assertj:assertj-core:3.1.0'
+  'org.beanshell:bsh:2.0b4'
+  'org.hamcrest:hamcrest-core:1.3'
+  'org.hamcrest:hamcrest-library:1.3'
+  'org.objenesis:objenesis:2.1'
+  'org.testng:testng:6.9.6'
+  'org.yaml:snakeyaml:1.15'
+
+  begin
+    i = 'javax.inject:javax.inject:1'
+    g, a, v = i.split(':')
+    c = API::MavenCentralRepository.new(g, a, v)
+
+    license_info = c.get_license_info
+    $plog.debug("license_info #{license_info}")
+
+    if license_info[:licenses].size == 1
+    elsif license_info[:licenses].size > 1
+      $plog.debug("#{g}:#{a}:#{v} -- has multi license")
+    end
+
+  rescue Exception => e
+    $plog.error("#{g}:#{a}:#{v}, #{e}")
+  end
+
+  $plog.info("results: #{license_info}")
+end

data/lib/api/mq.rb

@@ -0,0 +1,30 @@
+require 'bunny'
+
+module API
+
+class RabbitMQ
+  # "amqp://guest:guest@localhost:5672"
+  def initialize(rabbit_mq_conn_str)
+    @conn = Bunny.new(rabbit_mq_conn_str)
+    @conn.start
+    @ch = @conn.create_channel
+  end
+
+  def publish(queue_name, message, auto_release=true, durable=true, check_exist=false)
+    # TODO: @Micfan, 检查队列中是否存在该消息,如果已存在或正在处理,则返回false
+    def queue_exist(message)
+      false
+    end
+
+    q = @ch.queue(queue_name, :auto_release => auto_release, :durable => durable)
+    x = @ch.default_exchange
+
+    if check_exist and not queue_exist(message)
+      x.publish(message, :routing_key => q.name)
+    else
+      x.publish(message, :routing_key => q.name)
+    end
+  end
+end
+
+end

data/lib/api/npm_registry.rb

@@ -0,0 +1,169 @@
+require 'json'
+require 'open3'
+require 'httparty'
+
+require_relative '../../config/config'
+require_relative '../api/pattern'
+require_relative '../api/github'
+
+module API
+  # I have no time to research the npm cli source code(https://github.com/npm/npm-registry-client),
+  # quickly write this.
+  class NpmRegistry
+    def initialize(pack_name, pack_version)
+      @default_registry = 'http://registry.npmjs.org/'
+      @pack_name = pack_name
+      @pack_version = pack_version
+      @pack_meta = nil
+    end
+
+    # RESTful API: http://registry.npmjs.org/:pack_name
+    # TEST:        http://registry.npmjs.org/grunt
+    def get_package_meta
+      package_meta = nil
+      api_url = "#{@default_registry}#{@pack_name}"
+      $plog.debug(api_url)
+      response = HTTParty.get(api_url)
+      if response.code == 200
+        package_meta = JSON.parse(response.licenses)
+      elsif response.code == 404
+        $plog.error("!!! Npm registry API 404 Not found: #{response}")
+      else
+        $plog.error("!!! Npm registry API error response: #{response}")
+      end
+      package_meta
+    end
+
+    # RESTful API: http://registry.npmjs.org/grunt/?version=0.1.0
+    def get_package_info_by_version
+      package_info = nil
+      api_url = "#{@default_registry}#{@pack_name}/?version=#{@pack_version}"
+      $plog.debug(api_url)
+      response = HTTParty.get(api_url)
+      if response.code == 200
+        package_info = JSON.parse(response.licenses)
+      elsif response.code == 404
+        $plog.error("!!! Npm registry API 404 Not found: #{response}")
+      else
+        $plog.error("!!! Npm registry API error response: #{response}")
+      end
+      package_info
+    end
+
+    # DOC: https://www.npmjs.com/package/semver
+    # DOC: https://github.com/npm/node-semver
+    # sem_version_range: '~1.2.3'
+    def get_available_versions(sem_version_range)
+      available_versions = []
+      package_meta = get_package_meta
+      all_versions = package_meta['versions']
+      $plog.debug("sem_version_range: #{sem_version_range}")
+      all_versions.each_key {|version|
+        # node -e "var semver = require('semver'); var result = semver.satisfies('1.2.3', '1.x || >=2.5.0 || 5.0.0 - 7.2.3'); console.log(result);"
+        cmd = "node -e \"var semver = require('semver'); var available = semver.satisfies('#{version}', '#{sem_version_range}'); console.log(available);\""
+        # $plog.debug(cmd)
+        Open3.popen3(cmd) {|i,o,e,t|
+          out = o.readlines
+          error = e.readlines
+          if error.length > 0
+            $plog.error(error)
+            raise "node -e evaluate script error: #{cmd}, #{error}"
+          elsif out.length > 0
+            available = out[0].gsub(/\n/,'')
+            if available == 'true'
+              $plog.debug("available version: #{version}")
+              available_versions.push(version)
+            end
+          else
+            raise "node -e evaluate script error: #{cmd}, #{error}"
+          end
+        }
+      }
+      available_versions
+    end
+
+    def chose_one_available_version(sem_version_range)
+      available_versions = get_available_versions(sem_version_range)
+      # TODO: is it right?
+      $plog.debug("chosed version: #{available_versions.last}")
+      available_versions.last
+    end
+
+    def get_license_info()
+      license = nil
+      license_url = nil
+      license_text = nil
+      source_url = nil
+      homepage = nil
+
+      pack_info = get_package_info_by_version
+      $plog.debug("Npm registry pack_info: #{pack_info}")
+      if pack_info
+        licenses = pack_info['licenses']
+        if licenses != nil
+          # TODO: multi licenses:
+          # eg. http://registry.npmjs.org/grunt/?version=0.1.0
+          license = licenses[0]['type']
+
+          # TODO: convert http://github.com/cowboy/grunt/blob/master/LICENSE-MIT to raw
+          license_url = licenses[0]['url']
+          if license_url =~ API::SOURCE_URL_PATTERN[:github_html_page]
+            license_url, license_text = API::Github.convert_htmlpage_to_raw_url(license_url)
+          end
+        else
+          license = pack_info['license']
+        end
+
+        homepage = pack_info['homepage']
+        repository = pack_info['repository']
+        if repository
+          url = repository['url']
+          if repository['type'] == 'git'
+            source_url =
+                if url =~ /^git/
+                  url.gsub(/(git:\/\/|git\+ssh:\/\/git\@)/, 'http://').gsub(/\.git$/, '')
+                elsif url =~ /^http/
+                  url.gsub(/tree.*$/, '')
+                end
+          elsif repository['type'] =~ 'http'
+            source_url = repository['url'].gsub(/tree.*$/, '')
+          else
+            $plog.info("repository special type: #{repository}")
+          end
+        end
+
+        if license_text == nil
+          # TODO:
+        end
+      end
+      {
+        license: license,
+        license_url: license_url,
+        license_text: license_text,
+        source_url: source_url,
+        homepage: homepage
+      }
+    end
+  end
+end
+
+if __FILE__ == $0
+  # name = 'browserify'
+  # version = '12.0.1'
+  name = 'grunt'
+  version = '0.1.0'
+  n = API::NpmRegistry.new(name, version)
+  pack_info = n.get_package_info_by_version
+
+  license_info = n.get_license_info
+  $plog.info(license_info)
+
+  # pack_meta = n.get_package_meta
+  # p pack_meta
+  # p pack_meta['versions']
+
+  # sem_ver = '>=2.5.0'
+  # vs = n.get_available_versions(sem_ver)
+  # p vs
+end
+