license_auto 0.1.1.1

data/lib/parser/enums.rb

@@ -0,0 +1,161 @@
+##
+# Golang(v1.4.2 darwin/amd64) system libraries
+
+GOLANG_STD_PACKAGES = [
+'cmd/addr2line',
+'cmd/cgo',
+'cmd/fix',
+'cmd/go',
+'cmd/gofmt',
+'cmd/nm',
+'cmd/objdump',
+'cmd/pack',
+'cmd/pprof',
+'cmd/yacc',
+'archive/tar',
+'archive/zip',
+'bufio',
+'bytes',
+'cmd/internal/goobj',
+'cmd/internal/objfile',
+'cmd/pprof/internal/commands',
+'cmd/pprof/internal/driver',
+'cmd/pprof/internal/fetch',
+'cmd/pprof/internal/plugin',
+'cmd/pprof/internal/profile',
+'cmd/pprof/internal/report',
+'cmd/pprof/internal/svg',
+'cmd/pprof/internal/symbolizer',
+'cmd/pprof/internal/symbolz',
+'cmd/pprof/internal/tempfile',
+'compress/bzip2',
+'compress/flate',
+'compress/gzip',
+'compress/lzw',
+'compress/zlib',
+'container/heap',
+'container/list',
+'container/ring',
+'crypto',
+'crypto/aes',
+'crypto/cipher',
+'crypto/des',
+'crypto/dsa',
+'crypto/ecdsa',
+'crypto/elliptic',
+'crypto/hmac',
+'crypto/md5',
+'crypto/rand',
+'crypto/rc4',
+'crypto/rsa',
+'crypto/sha1',
+'crypto/sha256',
+'crypto/sha512',
+'crypto/subtle',
+'crypto/tls',
+'crypto/x509',
+'crypto/x509/pkix',
+'database/sql',
+'database/sql/driver',
+'debug/dwarf',
+'debug/elf',
+'debug/gosym',
+'debug/macho',
+'debug/pe',
+'debug/plan9obj',
+'encoding',
+'encoding/ascii85',
+'encoding/asn1',
+'encoding/base32',
+'encoding/base64',
+'encoding/binary',
+'encoding/csv',
+'encoding/gob',
+'encoding/hex',
+'encoding/json',
+'encoding/pem',
+'encoding/xml',
+'errors',
+'expvar',
+'flag',
+'fmt',
+'go/ast',
+'go/build',
+'go/doc',
+'go/format',
+'go/parser',
+'go/printer',
+'go/scanner',
+'go/token',
+'hash',
+'hash/adler32',
+'hash/crc32',
+'hash/crc64',
+'hash/fnv',
+'html',
+'html/template',
+'image',
+'image/color',
+'image/color/palette',
+'image/draw',
+'image/gif',
+'image/jpeg',
+'image/png',
+'index/suffixarray',
+'io',
+'io/ioutil',
+'log',
+'log/syslog',
+'math',
+'math/big',
+'math/cmplx',
+'math/rand',
+'mime',
+'mime/multipart',
+'net',
+'net/http',
+'net/http/cgi',
+'net/http/cookiejar',
+'net/http/fcgi',
+'net/http/httptest',
+'net/http/httputil',
+'net/http/internal',
+'net/http/pprof',
+'net/mail',
+'net/rpc',
+'net/rpc/jsonrpc',
+'net/smtp',
+'net/textproto',
+'net/url',
+'os',
+'os/exec',
+'os/signal',
+'os/user',
+'path',
+'path/filepath',
+'reflect',
+'regexp',
+'regexp/syntax',
+'runtime',
+'runtime/cgo',
+'runtime/debug',
+'runtime/pprof',
+'runtime/race',
+'sort',
+'strconv',
+'strings',
+'sync',
+'sync/atomic',
+'syscall',
+'testing',
+'testing/iotest',
+'testing/quick',
+'text/scanner',
+'text/tabwriter',
+'text/template',
+'text/template/parse',
+'time',
+'unicode',
+'unicode/utf16',
+'unicode/utf8',
+'unsafe']

data/lib/parser/gemfile_parser.rb

@@ -0,0 +1,221 @@
+require 'thread/pool'
+require 'bundler'
+
+require_relative './Obtain_path'
+require_relative './Ruby_extractor'
+require_relative './utils'
+require_relative '../config/config'
+require_relative '../lib/api/pattern'
+#require 'License_recognition'
+
+include Ruby_extractor
+
+
+
+module ExtractRuby
+  class Ex
+    include Ruby_extractor,Utils
+  end
+
+  class RubyExtractotr < Ex
+
+    # attr_reader :two_dependencies
+    attr_reader :package_list
+
+    def initialize(repo_path)
+      @repo_path = repo_path
+
+      @package_list       = [] # [name, version, status] package list , not exist "\n"
+      @rubygems_not_found = 30 # rubygemsDB not found
+      @status_init = 10
+      # gem server or git uri
+      @status_private_source = 33
+    end
+
+    # https://gist.github.com/flavio/1722530
+    # DOC: https://github.com/bundler/bundler/blob/master/lib/bundler/lockfile_parser.rb
+    # description : use bundler extract ruby package from gemfile.lcok
+    # repo_path   : local repo path , type : String
+    # st_true     : Correct extraction 10
+    # s.source.options['tag'], ['revision']
+    def get_first_level_specs()
+      all_specs = []
+      lockfiles = Obtain_path.new(@repo_path, "gem", ".lock").get_data
+      if lockfiles.size == 0
+        $plog.info('Gemfile.lock files not found, Ruby packages not found')
+        return all_specs
+      end
+
+      lockfiles.each do |pathname|
+        $plog.debug("Gemfile.lock file pathname: #{pathname}")
+        lockfile_parser = Bundler::LockfileParser.new(Bundler.read_file(pathname))
+        lockfile_parser.specs.each do |s|
+          all_specs.push(s)
+          # The 2nd level dependencies
+          # s.dependencies.each{|rows|
+          #   rows.requirement.requirements.each{|row|
+          #     tmp.concat [[row[0], row[1].version]]
+          #   }
+          # }
+        end
+      end
+      all_specs
+    end
+
+    def start
+      all_packs = []
+      require_relative '../lib/api/gem_data'
+      require_relative '../lib/api'
+      local_gemdb = API::GemData.new
+      rubygems_org = 'https://rubygems.org'
+      rubygems_org_domain = 'rubygems.org'
+
+      first_level_specs = get_first_level_specs
+      first_level_specs.each {|s|
+        # $plog.debug("git_uri: #{s.source.options['uri']}, remotes: #{s.source.options['remotes']}")
+        # gem_server_remotes.join(',').gsub(/http[s]?:\/\//, '')
+        pack = nil
+        pack_name = s.name
+        pack_version = s.version.to_s
+
+        if s.source.class == Bundler::Source::Git
+          source_url = s.source.options['uri']
+          _status = @status_init
+          _lang = source_url
+          _cmt = nil
+
+          if source_url =~ API::SOURCE_URL_PATTERN[:github]
+            g = API::Github.new(source_url)
+            source_url = g.repo_url
+            _lang = 'https://github.com'
+            if g.list_contents('').size == 0
+              _status = @status_private_source
+              _cmt = 'Private git uri, source code not found'
+            end
+          end
+          pack = {
+            pack_name: pack_name,
+            pack_version: pack_version,
+            homepage: nil,
+            source_url: source_url,
+            license: nil,
+            status: _status,
+            language: _lang,
+            # This cmt require local rubygems.org database always up-to-date
+            cmt: _cmt
+          }
+        elsif s.source.class == Bundler::Source::Rubygems
+          gem_server_remotes = s.source.options['remotes']
+
+          formatted_remotes = gem_server_remotes.collect {|r|
+            r.gsub(/\/$/, '').gsub(/http:\/\/rubygems\.org/, rubygems_org)
+          }
+          # TODO: get Gemfile source section
+          db_pack = local_gemdb.get_gem(pack_name, pack_version)
+          if formatted_remotes.index(rubygems_org) and db_pack
+            pack = db_pack.merge({
+                                    :status => 10,
+                                    :language => rubygems_org_domain,
+                                    :cmt => nil
+                                  })
+          else
+            formatted_remotes.delete(rubygems_org)
+            pack = {
+              pack_name: pack_name,
+              pack_version: pack_version,
+              homepage: nil,
+              source_url: nil,
+              license: 'UNKNOWN',
+              status: @status_private_source,
+              language: formatted_remotes.join(''),
+              # This cmt require local rubygems.org database always up-to-date
+              cmt: 'Private gem server or source code not found'
+            }
+          end
+        end
+        all_packs.push(pack)
+      }
+      all_packs
+    end
+
+    def select_rubygems_db
+      # FIXME: db connection
+      require_relative '../lib/api/gem_data'
+      rubygems_result = []
+      ruby_gems = API::GemData.new
+      @package_list.each do |row|
+        pack_hash = {
+            :pack_name => nil,
+            :pack_version => nil,
+            :homepage => nil,
+            :source_url => nil,
+            :license => nil,
+            :status => 10,
+            :cmt => nil,
+            :language => nil
+        }
+        if row['uri'] != nil
+          pack_hash[:language]     = row['uri']
+          pack_hash[:pack_name]    = row['name']
+          pack_hash[:pack_version] = row['version']
+          pack_hash[:source_url]   = row['uri']
+          rubygems_result << pack_hash
+          next
+        elsif row['remotes'] != nil
+          pack_hash[:language] = row['remotes']
+        end
+        if 10 == row['st']
+          result = ruby_gems.get_gemdata(row['name'],row['version'])
+          if result != nil
+            pack_hash[:pack_name]    = result[:name]
+            pack_hash[:pack_version] = result[:version]
+            pack_hash[:homepage]     = result[:homepage]
+            pack_hash[:source_url]   = result[:source_url]
+            pack_hash[:license]      = result[:license]
+            pack_hash[:status]       = row['st']
+          elsif nil == result
+            pack_hash[:pack_name]    = row['name']
+            if "" == row['version']
+              pack_hash[:pack_version] = nil
+            else
+              pack_hash[:pack_version] = row['version']
+            end
+            pack_hash[:status]       = @rubygems_not_found
+          end
+        end
+
+        rubygems_result << pack_hash
+      end
+
+      return rubygems_result
+    end
+
+  end
+end
+
+if __FILE__ == $0
+
+end
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

data/lib/parser/golang_parser.rb

@@ -0,0 +1,104 @@
+#!/usr/bin/env ruby
+# coding: utf-8
+
+require 'json'
+require 'open3'
+require 'set'
+require_relative '../../lib/cloner'
+require_relative '../../enums'
+require_relative '../../config/config'
+require_relative '../recorder'
+
+
+class GolangParser
+
+  def self.start(repo_filepath)
+    pack__name_uris = {}
+
+    pack_names = self.find_pack_names(repo_filepath)
+    pack_names.each {|pack_name|
+      pack_uri = ''
+      begin
+        pack_uri = self.golang_extpack_to_repo_uri(pack_name)
+      rescue Exception => e
+        # todo: save bad golang package into DB
+        pack_uri = e
+      end
+      name = pack_uri.gsub(/http(s)?:\/\//,'')
+      if pack_uri != nil
+        pack__name_uris.store(name, pack_uri)
+      end
+    }
+    $plog.debug("GolangParser: pack__name_uris: #{pack__name_uris}")
+    return pack__name_uris
+  end
+
+  # return: pack_names {Set} eg. { 'github.com/some_golang_author/hist_package', ... }
+  def self.find_pack_names(repo_filepath)
+    pack_names = Set.new
+
+    $plog.debug("repo_filepath: #{repo_filepath}")
+
+    Dir.chdir(repo_filepath) do
+      cmd = 'go list -json ./...'
+      Open3.popen3(cmd) {|i,o,e,t|
+        out = o.readlines
+        error = e.readlines
+        if error.length > 0 and error[0] != "warning: \"./...\" matched no packages\n"
+          raise "cmd error: #{error}"
+        elsif out.length > 0
+          out2 = out.join('').gsub(/}\n{/, "}\n\n{").split(/\n\n/)
+          out2.each {|s|
+            j = JSON::parse(s)
+            # $plog.info("json: #{j}")
+            imports = []
+            section_keys = ['Deps', 'Imports', 'TestImports', 'XTestImports']
+            section_keys.each {|s|
+              if j[s]
+                imports += j[s]
+              end
+            }
+
+            if imports.size == 0
+              next
+            else
+              imports.each {|d|
+                unless GOLANG_STD_PACKAGES.include?(d)
+                  unless d.index('.').nil?
+                    pack_names.add(d)
+                  end
+                end
+              }
+            end
+          }
+        else
+          $plog.error "!!!Opps'"
+        end
+        exit_status = t.value
+        pack_names
+        p pack_names
+      }
+    end
+    return pack_names
+  end
+
+  def self.golang_extpack_to_repo_uri(extpack)
+    pack_repo = nil
+    # todo: Case lower convert
+    uri = no_protocol_uri = extpack.split('/')
+    host, owner, pack_name = uri[0], uri[1], uri[2]
+    # todo: In china, you have to setup a proxy for Google Code
+    # sites = {
+    #     #  'code.google.com',
+    #   https: ['github.com', 'bitbucket.org', 'git.oschina.net'],
+    #   http: [],
+    #   git: []
+    # }
+    if extpack.index('.')
+      pack_repo = "https://#{host}/#{owner}/#{pack_name}"
+    else
+      $plog.error("不是外部Golang包: #{extpack}")
+    end
+    return pack_repo
+  end
+end