yavdb 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9cb5bd2ebaf5e3a0511328aa22b474bb22b9ce22
-  data.tar.gz: dea395b8c916a9b8d83560df597b03bcbb92bf8b
+  metadata.gz: f1f0cdfd79936efa10947a4b983c295f7059d930
+  data.tar.gz: 2ba6d7e40d9cb9d56249ff767ec5156520a610ae
 SHA512:
-  metadata.gz: 28e19a9021b471964697a9b2e4a9733c6dc05d2301e3fb0380fcd88853568568cdfde0395cf8f896b1e57f54b6bc040c2059a64dd1965bf64d84609a66caffc0
-  data.tar.gz: ff70031ab40e5492833fe28eb6dc72b03a69ef5aa0dc0ddab4c474d45750b693c3a821e523938a8d6b3ad8ab604e291353f7a4e6e88e8266fc61506dfcbaff51
+  metadata.gz: d29009672be459fc5c40701735d46f8460ccb3fad87169ecb1937a0858d6028cb008b8b2c4b1097758a418ea70c93ef959d0391f8e36cd729374b21afbdda3b7
+  data.tar.gz: ff5295068f3530f150004146154df3583bcce4a412d083181520b43a9b4e8d7401af7c330f4df6a85d325dc8cf0d47423d121de60949132ee8f9e805eb7c4fa4

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    yavdb (0.2.1)
+    yavdb (0.3.0)
       json (~> 2.1)
       kramdown (~> 1.17)
       oga (~> 2.15)
@@ -82,4 +82,4 @@ DEPENDENCIES
   yavdb!
 
 BUNDLED WITH
-   1.16.3
+   1.16.6

data/README.md

@@ -12,8 +12,8 @@ developers identify and fix know vulnerabilities in their apps.
 The sources for this database include 
 [Rubysec](https://rubysec.com/),
 [snyk](https://snyk.io/),
-[OSSIndex](https://ossindex.net/),
-[NodeSecurity](https://nodesecurity.io/),
+[OSSIndex (deprecated)](https://ossindex.net/),
+[NodeSecurity (deprecated)](https://nodesecurity.io/),
 [Friends of PHP](https://github.com/FriendsOfPHP/security-advisories),
 [Magento Related Security Advisories](https://github.com/victims/victims-cve-db),
 [Victims CVE Database](https://github.com/victims/victims-cve-db)
@@ -25,11 +25,9 @@ The sources for this database include
 ## Installation
 
 ```sh
-gem install yavdb --pre
+gem install yavdb
 ```
 
-> Notice the `--pre` in the end
-
 ## TODO:
 
 #### Tests
@@ -38,7 +36,6 @@ gem install yavdb --pre
     - [ ] [Rubysec](lib/yavdb/sources/ruby_advisory.rb)
     - [X] [snyk](lib/yavdb/sources/snyk_io.rb)
     - [ ] [OSSIndex](lib/yavdb/sources/ossindex.rb)
-    - [X] [NodeSecurity](lib/yavdb/sources/nodesecurity_io.rb)
     - [ ] [Friends of PHP and Magento Related Security Advisories](lib/yavdb/sources/friends_of_php.rb)
     - [ ] [Victims CVE Database](lib/yavdb/sources/victims.rb)
 * Others

data/lib/yavdb/database.rb

@@ -60,12 +60,18 @@ module YAVDB
       def save_to_file(database_path, vulns)
         vulns.map do |package_manager, vunerabilities_by_pm|
           vunerabilities_by_pm.map do |package, vunerabilities_by_p|
+            previous_vulnerabilities = search(database_path, package_manager, package)
+
             package_path           = package_path(database_path, package_manager, package)
             package_path_directory = File.dirname(package_path)
             FileUtils.mkdir_p(package_path_directory) unless File.exist?(package_path_directory)
 
+            uniq_vunerabilities_by_p = Hash[previous_vulnerabilities.concat(vunerabilities_by_p).map { |vuln| [vuln.id, vuln] }].values
+
+            next unless uniq_vunerabilities_by_p.any?
+
             File.open(package_path, 'wb') do |file|
-              package_vulns_yml_str = vunerabilities_by_p
+              package_vulns_yml_str = uniq_vunerabilities_by_p
                                         .sort_by(&:id)
                                         .map(&:to_map)
                                         .to_yaml(

data/lib/yavdb/dtos/advisory.rb

@@ -79,9 +79,9 @@ module YAVDB
     def to_map
       map = {}
       members.each do |m|
-        next unless self[m] && (
-        (self[m].is_a?(String) && !self[m].empty?) ||
-          (self[m].is_a?(Array) && self[m].any?))
+        next if !self[m] ||
+                (self[m].is_a?(String) && self[m].empty?) ||
+                (self[m].is_a?(Array) && self[m].none?)
 
         map[m.to_s] = self[m] if self[m]
       end

data/lib/yavdb/sources/friends_of_php.rb

@@ -54,9 +54,15 @@ module YAVDB
             info['versions'].join(' ')
           end.flatten
 
+          cves = [advisory_hash['cve']].reject { |cve| cve == '~' }
+
           package_name = advisory_hash['reference'].gsub(%r{composer:\/\/(.*)}, '\1')
+
+          vuln_id_stamp = (cves && cves[0]) || date
+          vuln_id       = "friendsofphp:packagist:#{package_name}:#{vuln_id_stamp}"
+
           YAVDB::Advisory.new(
-            "friendsofphp:packagist:#{package_name}:#{date}",
+            vuln_id,
             advisory_hash['title'],
             nil, #:description
             package_name,
@@ -65,7 +71,7 @@ module YAVDB
             nil, #:patched_versions
             nil, #:severity
             PACKAGE_MANAGER,
-            [advisory_hash['cve']].reject { |cve| cve == '~' },
+            cves,
             nil, #:cwe
             nil, #:osvdb
             nil, #:cvss_v2_vector

data/lib/yavdb/sources/ossindex.rb

@@ -95,8 +95,11 @@ module YAVDB
                          .reject { |v| v == '-' }
             versions = ['*'] unless versions.any?
 
+            vuln_id_stamp = (cve && cve[0]) || published_date
+            vuln_id       = "ossindex:#{package_manager}:#{package_name}:#{vuln_id_stamp}"
+
             YAVDB::Advisory.new(
-              "ossindex:#{package_manager}:#{package_name}:#{published_date}",
+              vuln_id,
               advisory['title'],
               advisory['description'],
               package_name,

data/lib/yavdb/sources/ruby_advisory.rb

@@ -54,8 +54,11 @@ module YAVDB
                                     ['*']
                                   end
 
+            vuln_id_stamp = (cve && cve[0]) || date
+            vuln_id       = "rubyadvisory:rubygems:#{advisory_hash['gem']}:#{vuln_id_stamp}"
+
             YAVDB::Advisory.new(
-              "rubyadvisory:rubygems:#{advisory_hash['gem']}:#{date}",
+              vuln_id,
               advisory_hash['title'],
               advisory_hash['description'],
               advisory_hash['gem'],

data/lib/yavdb/sources/snyk_io.rb

@@ -119,8 +119,13 @@ module YAVDB
                                    published_date
                                  end
 
+            vuln_id_stamp = (sidebar_data[:cve] && sidebar_data[:cve][0]) ||
+                            sidebar_data[:id].split(%r{-|:}).last ||
+                            disclosed_date
+            vuln_id       = "snykio:#{package_manager}:#{affected_package}:#{vuln_id_stamp}"
+
             YAVDB::Advisory.new(
-              "snykio:#{package_manager}:#{affected_package}:#{disclosed_date}",
+              vuln_id,
               title,
               body_data[:description],
               affected_package,
@@ -157,7 +162,6 @@ module YAVDB
                 last_elem        = description_sections.last
                 new_body         = last_elem[:body].push(field)
                 last_elem[:body] = new_body
-                description_sections.push(last_elem)
               end
             end
 

data/lib/yavdb/sources/victims.rb

@@ -59,8 +59,11 @@ module YAVDB
 
           def create(advisory_hash, language, url)
             advisory_hash['affected'].map do |affected_package|
+              vuln_id_stamp = advisory_hash['cve'] || 'date'
+              vuln_id       = "victims:#{language.package_manager}:#{language.name_parser[affected_package]}:#{vuln_id_stamp}"
+
               YAVDB::Advisory.new(
-                "victims:#{language.package_manager}:#{language.name_parser[affected_package]}:date",
+                vuln_id,
                 advisory_hash['title'],
                 advisory_hash['description'],
                 language.name_parser[affected_package],

data/lib/yavdb/version.rb

@@ -16,6 +16,6 @@
 
 module YAVDB
 
-  VERSION = '0.2.1'
+  VERSION = '0.3.0'
 
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: yavdb
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Rodrigo Fernandes
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-10-17 00:00:00.000000000 Z
+date: 2018-10-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -227,7 +227,6 @@ files:
 - lib/yavdb/dtos/advisory.rb
 - lib/yavdb/source_types/git_repo.rb
 - lib/yavdb/sources/friends_of_php.rb
-- lib/yavdb/sources/nodesecurity_io.rb
 - lib/yavdb/sources/ossindex.rb
 - lib/yavdb/sources/ruby_advisory.rb
 - lib/yavdb/sources/snyk_io.rb

data/lib/yavdb/sources/nodesecurity_io.rb

@@ -1,120 +0,0 @@
-# yavdb - The Free and Open Source vulnerability database
-# Copyright (C) 2017-present Rodrigo Fernandes
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-require 'oga'
-require 'oga/xml/entities'
-require 'date'
-
-require_relative '../dtos/advisory'
-require_relative '../utils/http'
-
-module YAVDB
-  module Sources
-    module NodeSecurityIO
-      class Client
-
-        API_URL     = 'https://api.nodesecurity.io/advisories'
-        WEBSITE_URL = 'https://nodesecurity.io/advisories'
-
-        def self.advisories
-          fetch_advisories.map do |advisory_hash|
-            create(advisory_hash)
-          end
-        end
-
-        class << self
-
-          private
-
-          def fetch_advisories
-            offset     = 0
-            advisories = []
-
-            loop do
-              nodesecurity    = YAVDB::Utils::HTTP.get_page_contents("#{API_URL}?offset=#{offset}", false, 'nodesecurity.io/advisories')
-              advisories_json = JSON.parse(nodesecurity.join)
-
-              advisories_json['count'].positive? ? advisories = advisories.concat(advisories_json['results']) : break
-
-              offset += advisories_json['count']
-            end
-
-            advisories
-          end
-
-          def create(advisory_hash)
-            publish_date = Date.parse(advisory_hash['publish_date'])
-            created_at   = Date.parse(advisory_hash['created_at'])
-            updated_at   = Date.parse(advisory_hash['updated_at'])
-
-            vulnerable_versions =
-              if advisory_hash['vulnerable_versions'].nil? || advisory_hash['vulnerable_versions'].empty?
-                '*'
-              else
-                advisory_hash['vulnerable_versions']
-              end
-
-            YAVDB::Advisory.new(
-              "nodesecurity:npm:#{advisory_hash['module_name']}:#{publish_date}",
-              advisory_hash['title'],
-              advisory_hash['overview'],
-              advisory_hash['module_name'],
-              [vulnerable_versions],
-              nil, #:unaffected_versions
-              advisory_hash['patched_versions'],
-              severity(advisory_hash['cvss_score']),
-              'npm',
-              advisory_hash['cves'],
-              nil, #:cwe
-              nil, #:osvdb
-              nil, #:cvss_v2_vector
-              nil, #:cvss_v2_score
-              advisory_hash['cvss_vector'],
-              advisory_hash['cvss_score'],
-              publish_date,
-              created_at,
-              updated_at,
-              [advisory_hash['author']],
-              clean_references(advisory_hash['references']),
-              "#{WEBSITE_URL}/#{advisory_hash['id']}"
-            )
-          end
-
-          def severity(cvss_score)
-            case cvss_score
-              when 0.0..3.3 then
-                'low'
-              when 3.3..6.6 then
-                'medium'
-              else
-                'high'
-            end
-          end
-
-          def clean_references(references)
-            if references
-              [references.gsub(%r{.*?(http.+)}, '\1')]
-            else
-              []
-            end
-          end
-
-        end
-
-      end
-    end
-  end
-end