yavdb 0.6.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5bd83226987ff17616cf7fc728a9804295402491da0254c04f09e83b162070fd
-  data.tar.gz: '0629122cc39f6740a833aa61cb3da32acfcd9205ad1ed0658ac17380d2373874'
+  metadata.gz: e1e5f9733916e6d5f5d5be22dc13f15879d2d27c4d67f3a1da2354108de7c693
+  data.tar.gz: 8748dccfdbc49a566c1bc46d692b96d8dff6bbf17aa77b876fc20fd92066ae61
 SHA512:
-  metadata.gz: 14f2363effd7b653ad4eb9ca71a54236304d7b5e42d224ff3dfbc554dcef8fa8c76d0a19f8a7c5fed5b795199fde239cee6b7296e716494055f504ab0edb2520
-  data.tar.gz: 1a0ba50cbdf9c1b4fff512465886924458de19195e5e6fc03b1dbc5620c4e9ce2728bc7864d2830473748c68cc1225ef5669358a0c21da78f17e0ff12993071b
+  metadata.gz: 7be1bd86bf8f7c066cc4ee6e5cbc005d226bc70e0d1a8a0f849ac447c4c64032608cfef4a25abe46bdc297e985a1050450b355014113c25efd9bfc1afe4bccf2
+  data.tar.gz: 5e44694faa7f72af21f2cb2548e3059e7ab993872e0f1ad0b08d6f60f1f2bcef28cd7af02f9fb61a9ef20ae9ca18b6bd0172afe175cb8d77e78f1450bd2b2b47

data/.rubocop.yml

@@ -163,8 +163,6 @@ Lint/UselessAssignment:
 
 Lint/SuppressedException:
   Enabled: true
-  Exclude:
-  - "lib/yavdb/sources/snyk_io.rb"
 
 Metrics/AbcSize:
   Enabled: true

data/.ruby-version

@@ -1 +1 @@
-2.5.5
+2.5.7

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    yavdb (0.6.0)
+    yavdb (0.7.0)
       execjs (~> 2.7)
       json (~> 2.2)
       kramdown (~> 2.3)
@@ -133,4 +133,4 @@ DEPENDENCIES
   yavdb!
 
 BUNDLED WITH
-   2.1.2
+   2.2.5

data/README.md

@@ -11,7 +11,7 @@ developers identify and fix know vulnerabilities in their apps.
 
 The sources for this database include 
 [Rubysec](https://rubysec.com/),
-[snyk](https://snyk.io/),
+~~[snyk](https://snyk.io/),~~ (removed)
 [Friends of PHP](https://github.com/FriendsOfPHP/security-advisories),
 [Magento Related Security Advisories](https://github.com/victims/victims-cve-db),
 [Victims CVE Database](https://github.com/victims/victims-cve-db),
@@ -33,7 +33,6 @@ gem install yavdb
     
 * Sources
     - [ ] [Rubysec](lib/yavdb/sources/ruby_advisory.rb)
-    - [X] [snyk](lib/yavdb/sources/snyk_io.rb)
     - [ ] [OSSIndex](lib/yavdb/sources/ossindex.rb)
     - [ ] [Friends of PHP and Magento Related Security Advisories](lib/yavdb/sources/friends_of_php.rb)
     - [ ] [Victims CVE Database](lib/yavdb/sources/victims.rb)

data/lib/yavdb/version.rb

@@ -16,6 +16,6 @@
 
 module YAVDB
 
-  VERSION = '0.6.0'
+  VERSION = '0.7.0'
 
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: yavdb
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.7.0
 platform: ruby
 authors:
 - Rodrigo Fernandes
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-10-17 00:00:00.000000000 Z
+date: 2021-01-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: codacy-coverage
@@ -296,7 +296,6 @@ files:
 - lib/yavdb/sources/npmjs.rb
 - lib/yavdb/sources/ruby_advisory.rb
 - lib/yavdb/sources/rustsec.rb
-- lib/yavdb/sources/snyk_io.rb
 - lib/yavdb/sources/victims.rb
 - lib/yavdb/utils/cache.rb
 - lib/yavdb/utils/exec.rb

data/lib/yavdb/sources/snyk_io.rb

@@ -1,309 +0,0 @@
-# yavdb - The Free and Open Source vulnerability database
-# Copyright (C) 2017-present Rodrigo Fernandes
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-require 'oga'
-require 'oga/xml/entities'
-require 'kramdown'
-
-require_relative '../dtos/advisory'
-require_relative '../utils/http'
-
-module YAVDB
-  module Sources
-    module SnykIO
-      class Client
-
-        BASE_URL      = 'https://snyk.io'
-        BASE_VULN_URL = "#{BASE_URL}/vuln"
-        INFO_SEP      = '#=#'
-
-        PACKAGE_MANAGERS = ['composer', 'golang', 'maven', 'npm', 'nuget', 'pip', 'rubygems', 'cocoapods'].freeze
-
-        PACKAGE_MANAGER_ALIAS = Hash[
-          'composer' => 'packagist',
-          'go' => 'go',
-          'maven' => 'maven',
-          'npm' => 'npm',
-          'nuget' => 'nuget',
-          'pip' => 'pypi',
-          'rubygems' => 'rubygems',
-          'cocoapods' => 'cocoapods'
-        ].freeze
-
-        def self.advisories
-          urls = fetch_advisory_urls
-          urls.map do |advisory_url|
-            advisory_page = get_page_html(advisory_url, true, 'snyk.io/advisories')
-            create(advisory_url, advisory_page)
-          end.reject(&:nil?)
-        end
-
-        class << self
-
-          private
-
-          def fetch_advisory_urls
-            PACKAGE_MANAGERS.map do |pm|
-              fetch_advisory_recursive("#{BASE_VULN_URL}?type=#{pm}")
-            end.flatten
-          end
-
-          def fetch_advisory_recursive(page_url)
-            snykio = get_page_html(page_url, false, 'snyk.io/feed')
-
-            page_vuln_urls = snykio
-                               .css('table tbody tr td span a')
-                               .map { |anchor| anchor.get('href') }
-                               .map { |link| link if %r{\/vuln\/.+}.match?(link) }.compact
-
-            next_urls = if page_vuln_urls.any?
-                          next_url = snykio.css('a.pagination__next')
-                          if next_url&.first
-                            fetch_advisory_recursive(next_url.first.get('href'))
-                          else
-                            []
-                          end
-                        else
-                          []
-                        end
-
-            page_vuln_urls
-              .concat(next_urls)
-              .map do |url|
-              full_url = url
-              full_url = "#{BASE_URL}#{url}" unless url.start_with?('http')
-              full_url
-            end
-          end
-
-          def create(advisory_url, advisory_page)
-            severity = advisory_page.css('span.label__text').text.gsub(%r{(.*?) severity}, '\1')
-
-            package_manager = advisory_page.css('.breadcrumbs__list-item')[1].text.gsub(%r{\s+}, '').downcase
-            package_manager = PACKAGE_MANAGER_ALIAS[package_manager] || return
-
-            title = utf8(advisory_page.css('h1.header__title span.header__title__text').text)
-
-            affected_package = advisory_page.css('.custom-package-name').text
-            affected_package = advisory_page.css('.header__lede .breadcrumbs__list-item__link').text if affected_package.empty?
-
-            vulnerable_versions = (advisory_page.css('.custom-affected-versions') ||
-              advisory_page.css('.header__lede strong').drop(1).first).text.strip
-            vulnerable_versions = if vulnerable_versions.empty? || vulnerable_versions == 'ALL' || vulnerable_versions == '(,)'
-                                    ['*']
-                                  elsif ['maven', 'nuget', 'pypi'].include?(package_manager)
-                                    [vulnerable_versions]
-                                  else
-                                    [vulnerable_versions.tr(',', ' ')]
-                                  end
-
-            sidebar_data = parse_side_bar(advisory_page)
-            body_data    = parse_body(advisory_page)
-
-            published_date     = parse_date(sidebar_data[:published_date].to_s)
-            disclosed_date     = parse_date(sidebar_data[:disclosed_date].to_s) || published_date
-            last_modified_date = if sidebar_data[:last_modified_date]
-                                   parse_date(sidebar_data[:last_modified_date].to_s)
-                                 else
-                                   published_date
-                                 end
-
-            vuln_id_stamp = sidebar_data[:id].split(%r{-|:}).last || disclosed_date
-            vuln_id       = "snykio:#{package_manager}:#{affected_package}:#{vuln_id_stamp}"
-
-            YAVDB::Advisory.new(
-              vuln_id,
-              title,
-              body_data[:description],
-              affected_package,
-              vulnerable_versions,
-              nil, #:unaffected_versions
-              nil, #:patched_versions
-              severity,
-              package_manager,
-              sidebar_data[:cve],
-              sidebar_data[:cwe],
-              nil, #:osvdb
-              nil, #:cvss_v2_vector
-              nil, #:cvss_v2_score
-              nil, #:cvss_v3_vector
-              nil, #:cvss_v3_score
-              disclosed_date,
-              published_date,
-              last_modified_date,
-              [sidebar_data[:credit]].flatten,
-              body_data[:references],
-              advisory_url
-            )
-          end
-
-          def parse_body(advisory_page)
-            data = {}
-
-            description_sections = []
-            overview_fields      = advisory_page.css('.card.card--markdown .card__content > *')
-            overview_fields.each do |field|
-              if field.name == 'h2'
-                description_sections.push(:header => field, :body => [])
-              elsif description_sections.any?
-                last_elem        = description_sections.last
-                new_body         = last_elem[:body].push(field)
-                last_elem[:body] = new_body
-              end
-            end
-
-            description_sections.map do |section|
-              header = section[:header]
-              body   = section[:body]
-
-              case header.text
-                when %r{^(Overview|Details)$} then
-                  overview_str = body
-                                   .map(&:to_xml)
-                                   .map { |e| e.force_encoding('UTF-8') }
-                                   .join("\n")
-                  begin
-                    if data[:description]
-                      data[:description] += '\n'
-                    else
-                      data[:description] = ''
-                    end
-
-                    data[:description] += utf8(Kramdown::Document.new(overview_str, :html_to_native => true).to_kramdown)
-                  rescue StandardError
-                    # ignore
-                  end
-                when 'References' then
-                  references = []
-                  if body.any?
-                    body.first.css('li a').map do |elem|
-                      references.push(elem.get('href'))
-                    end
-                  end
-                  data[:references] = references.flatten
-              end
-            end
-
-            data
-          end
-
-          def parse_side_bar(advisory_page)
-            data = {}
-
-            advisory_page.css('.l-col .card .card__content dl > *').each_slice(2).to_a.map do |key, value|
-              case key.text
-                when 'Credit'
-                  data[:credit] = utf8(value.text.split(',').map { |str| str.strip.sub(%r{-\s*}, '') }.reject(&:empty?))
-                when 'CVE'
-                  data[:cve] = value.css('a').map { |a| a.text.strip.split(',') }.flatten.map(&:strip).reject(&:empty?)
-                when 'CWE'
-                  data[:cwe] = value.css('a').map { |a| a.text.strip.split(',') }.flatten.map(&:strip).reject(&:empty?)
-                when 'Snyk ID'
-                  data[:id] = value.text.strip
-                when 'Disclosed'
-                  data[:disclosed_date] = value.text.strip
-                when 'Published'
-                  data[:published_date] = value.text.strip
-                when 'Last modified'
-                  data[:last_modified_date] = value.text.strip
-              end
-            end
-
-            data
-          end
-
-          def get_page_html(source_url, with_cache, group_cache_key)
-            source_url = "#{BASE_URL}#{source_url}" unless source_url.start_with?('http')
-            body_lines = YAVDB::Utils::HTTP.get_page_contents(source_url, with_cache, group_cache_key)
-            body_lines = escape_vulnerable_versions(body_lines)
-            Oga.parse_html(body_lines, :strict => true)
-          end
-
-          def clean_references(references)
-            references.map do |reference|
-              reference
-                .gsub(%r{\s*-\s*(.*)}, '\1')
-                .gsub(%r{\[.+?\]\((.*)\).*}, '\1')
-                .strip
-            end
-          end
-
-          def parse_date(date_str)
-            Date.strptime(date_str, '%d %b, %Y')
-          rescue ArgumentError
-            # ignore
-          end
-
-          # HACK: Page contains non UTF-8 characters and we need to fix it to be able to convert to json
-          def utf8(value)
-            if value.is_a?(Array)
-              value.map { |sub_value| utf8(sub_value) }
-            elsif value.is_a?(String)
-              value.force_encoding('UTF-8')
-            else
-              value
-            end
-          end
-
-          # HACK: Page contains invalid HTML and we need to fix it to get the affected version
-          def escape_vulnerable_versions(body_lines)
-            cleaning = false
-            body_lines.map do |line|
-              if line.include?('<h2 id="overview">Overview</h2>')
-                cleaning = true
-              elsif line.include?('<h2 id=')
-                cleaning = false
-              end
-
-              if cleaning
-                key  = '<script>'
-                line = line.gsub(key, Oga::XML::Entities.encode(key))
-              end
-
-              if line.include?(', versions')
-                extracted     = line.gsub(%r{\s*<strong\s*>(.*)<\/strong>\s*}, '\1')
-                                  .gsub(%r{\s*<a.*>(.*)<\/a><\/strong>\s*}, '\1 ')
-                                  .gsub(%r{\s*<\/p>\s*}, '')
-                                  .gsub(%r{\s*<\/strong>\s*}, ' ')
-                                  .gsub(%r{\s*(.*)\s*}, '\1')
-                                  .gsub(%r{(\S*)(?:\s+.*)?,\s*versions\s*(.*)\s*}, "\\1#{INFO_SEP}\\2")
-                                  .tr("\n", ' ')
-                                  .gsub('&nbsp;', ' ')
-                                  .split(INFO_SEP)
-                fixed_version = Oga::XML::Entities.encode(extracted[1])
-                "</strong><span class=\"custom-package-name\">#{extracted[0]}</span><span class=\"custom-affected-versions\">#{fixed_version}</span>"
-              elsif line.include?(', <strong >ALL</strong> versions')
-                extracted = line.gsub(%r{\s*<strong\s*>(.*)<\/strong>\s*}, '\1')
-                              .gsub(%r{\s*<a.*>(.*)<\/a><\/strong>\s*}, '\1 ')
-                              .gsub(%r{\s*<\/p>\s*}, '')
-                              .gsub(%r{\s*<\/strong>\s*}, ' ')
-                              .gsub(%r{\s*(.*)\s*}, '\1')
-                              .gsub(%r{(\S*)(?:\s+.*)?,\s*.*\s*versions\s*}, '\1')
-                              .tr("\n", ' ')
-                              .split(INFO_SEP)
-                "</strong><span class=\"custom-package-name\">#{extracted[0]}</span><span class=\"custom-affected-versions\">*</span>"
-              else
-                line
-              end
-            end
-          end
-
-        end
-
-      end
-    end
-  end
-end