yavdb 0.4.4 → 0.4.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9f0547b87d256354a1353e774f878fe8f750e6c2
-  data.tar.gz: cb90e74feaea0a417dc28e28471bfd8be630fff6
+  metadata.gz: a18c36ce6f55c809ffc97c80f55090ae228aac37
+  data.tar.gz: 54780f521f02012b41710ce2d960295ef6ebaefa
 SHA512:
-  metadata.gz: 184b6626614ccde3b38cc486414ec98303aae946476acc4b98f624bec026ed96b8c651afc29b833f31625e1b7a5f7b41180f759f58a8cac1d498c194101e0055
-  data.tar.gz: 98cc1793d053550d58f90807a76a66197d9da2432308cf8ada6326bb84bd83d82e08bd00dd30e6f73b3c7759da2d263867a43375397b616392ee6b381763d84c
+  metadata.gz: 15a22f778a8a03760996abd841cfdf63736985fec74ee670fc2737a39930539c2dbda06ce13fe222ad39263dfe18a7f0885fb3ad69b1d154f9773bc027533c54
+  data.tar.gz: 85f4cfe790a5c2a39967fda83d8b2f414c060ee96153edf4c3b18e63b40861b0b851176e084940decdd879a998d38af3d74aa0c83c95abd66ff53f8ba03bf8e0

data/Gemfile.lock

@@ -1,11 +1,13 @@
 PATH
   remote: .
   specs:
-    yavdb (0.4.4)
+    yavdb (0.4.5)
+      execjs (~> 2.7.0)
       json (~> 2.1)
       kramdown (~> 1.17)
       oga (~> 2.15)
       semantic_interval (~> 0.1)
+      therubyracer (~> 0.12)
       thor (~> 0.20)
 
 GEM
@@ -17,9 +19,11 @@ GEM
       simplecov
     diff-lcs (1.3)
     docile (1.3.1)
+    execjs (2.7.0)
     jaro_winkler (1.5.1)
     json (2.1.0)
     kramdown (1.17.0)
+    libv8 (3.16.14.19)
     oga (2.15)
       ast
       ruby-ll (~> 2.1)
@@ -29,6 +33,7 @@ GEM
     powerpack (0.1.2)
     rainbow (3.0.0)
     rake (12.3.1)
+    ref (2.0.0)
     rspec (3.8.0)
       rspec-core (~> 3.8.0)
       rspec-expectations (~> 3.8.0)
@@ -64,7 +69,10 @@ GEM
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.2)
-    thor (0.20.0)
+    therubyracer (0.12.3)
+      libv8 (~> 3.16.14.15)
+      ref
+    thor (0.20.3)
     unicode-display_width (1.4.0)
 
 PLATFORMS
@@ -82,4 +90,4 @@ DEPENDENCIES
   yavdb!
 
 BUNDLED WITH
-   1.16.6
+   1.17.1

data/README.md

@@ -13,7 +13,6 @@ The sources for this database include
 [Rubysec](https://rubysec.com/),
 [snyk](https://snyk.io/),
 [OSSIndex (deprecated)](https://ossindex.net/),
-[NodeSecurity (deprecated)](https://nodesecurity.io/),
 [Friends of PHP](https://github.com/FriendsOfPHP/security-advisories),
 [Magento Related Security Advisories](https://github.com/victims/victims-cve-db),
 [Victims CVE Database](https://github.com/victims/victims-cve-db)

data/lib/yavdb/sources/friends_of_php.rb

@@ -39,7 +39,7 @@ module YAVDB
                 file_paths.map do |file_path|
                   advisory_hash = YAML.load_file(file_path)
                   url           = "#{repository_url}/blob/master/#{file_path}"
-                  filename      = File.basename(file_path, '.yml')
+                  filename      = File.basename(file_path, '.yaml')
                   create(url, filename, advisory_hash)
                 end
               end
@@ -59,8 +59,7 @@ module YAVDB
 
           package_name = advisory_hash['reference'].gsub(%r{composer:\/\/(.*)}, '\1')
 
-          vuln_id_stamp = (cves && cves[0]) || filename
-          vuln_id       = "friendsofphp:packagist:#{package_name}:#{vuln_id_stamp}"
+          vuln_id = "friendsofphp:packagist:#{package_name}:#{filename}"
 
           YAVDB::Advisory.new(
             vuln_id,

data/lib/yavdb/sources/npmjs.rb

@@ -0,0 +1,129 @@
+# yavdb - The Free and Open Source vulnerability database
+# Copyright (C) 2017-present Rodrigo Fernandes
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'execjs'
+require 'oga'
+require 'oga/xml/entities'
+
+require_relative '../dtos/advisory'
+require_relative '../utils/http'
+
+module YAVDB
+  module Sources
+    module NPMJS
+      class Client
+
+        API_URL = 'https://www.npmjs.com'
+
+        def self.advisories
+          packages = fetch_packages_recursive(0)
+          parse_vulnerabilities(packages)
+        end
+
+        class << self
+
+          private
+
+          def fetch_packages_recursive(page_number)
+            page = get_page_html(get_page_url(page_number), false, 'npmjs/feed')
+
+            script_tag = page.css('script').find { |script| script.text.include?('window.__context__') }.text
+            context = ExecJS.compile("var window = {};\n#{script_tag.force_encoding('utf-8')};")
+            advisory_data = context.exec('return window.__context__.context.advisoriesData')
+
+            packages = advisory_data['objects']
+
+            next_url = advisory_data['urls']['next']
+            next_packages = if next_url && !next_url&.include?("page=#{page_number}")
+                              fetch_packages_recursive(page_number + 1)
+                            else
+                              []
+                            end
+
+            packages.concat(next_packages)
+          end
+
+          def parse_vulnerabilities(packages)
+            packages.map { |package| create(package) }.flatten
+          end
+
+          def create(package)
+            published_date = Date.strptime(package['created'], '%s')
+            updated_date = Date.strptime(package['updated'], '%s')
+
+            cves = package['cves'] || []
+
+            versions = [package['vulnerable_versions']]
+            versions = ['*'] unless versions.any?
+
+            vuln_id = "npmjs:npm:#{package['module_name']}:#{package['id']}"
+
+            YAVDB::Advisory.new(
+              vuln_id,
+              package['title'],
+              package['overview'],
+              package['module_name'],
+              versions,
+              nil, #:unaffected_versions
+              nil, #:patched_versions
+              parse_severity(package['severity']),
+              'npm',
+              cves,
+              package['cwe'],
+              nil, #:osvdb
+              nil, #:cvss_v2_vector
+              nil, #:cvss_v2_score
+              nil, #:cvss_v3_vector
+              nil, #:cvss_v3_score
+              published_date,
+              published_date,
+              updated_date,
+              package['found_by']['name'],
+              package['url'],
+              package['url']
+            )
+          end
+
+          def get_page_html(source_url, with_cache, group_cache_key)
+            body_lines = YAVDB::Utils::HTTP.get_page_contents(source_url, with_cache, group_cache_key)
+            Oga.parse_html(body_lines, :strict => true)
+          end
+
+          def get_page_url(page)
+            "#{API_URL}/advisories?page=#{page}&perPage=300&order=-id"
+          end
+
+          def parse_severity(severity)
+            case severity
+              when 'low' then
+                'low'
+              when 'moderate' then
+                'medium'
+              when 'high' then
+                'high'
+              when 'critical' then
+                'high'
+              else
+                'high'
+            end
+          end
+
+        end
+
+      end
+    end
+  end
+end

data/lib/yavdb/sources/ossindex.rb

@@ -96,8 +96,7 @@ module YAVDB
                          .map { |version| version.gsub("''", '') }
             versions = ['*'] unless versions.any?
 
-            vuln_id_stamp = (cve && cve[0]) || published_date
-            vuln_id       = "ossindex:#{package_manager}:#{package_name}:#{vuln_id_stamp}"
+            vuln_id = "ossindex:#{package_manager}:#{package_name}:#{advisory['id']}"
 
             YAVDB::Advisory.new(
               vuln_id,

data/lib/yavdb/sources/ruby_advisory.rb

@@ -43,20 +43,19 @@ module YAVDB
 
           private
 
-          def create(_file_path, advisory_hash)
+          def create(file_path, advisory_hash)
             date                = Date.strptime(advisory_hash['date'].to_s, '%Y-%m-%d')
             severity            = severity(advisory_hash['cvss_v2'], advisory_hash['cvss_v3'])
             cve                 = advisory_hash['cve'] && "CVE-#{advisory_hash['cve']}"
-            osvdb               = advisory_hash['osvdb'] && "OSVDB-#{advisory_hash['osvdb']}"
             references          = references(advisory_hash)
+            filename            = File.basename(file_path, '.yml')
             vulnerable_versions = if advisory_hash['unaffected_versions'] || advisory_hash['patched_versions']
                                     nil
                                   else
                                     ['*']
                                   end
 
-            vuln_id_stamp = cve || osvdb || date
-            vuln_id       = "rubyadvisory:rubygems:#{advisory_hash['gem']}:#{vuln_id_stamp}"
+            vuln_id = "rubyadvisory:rubygems:#{advisory_hash['gem']}:#{filename}"
 
             YAVDB::Advisory.new(
               vuln_id,

data/lib/yavdb/sources/snyk_io.rb

@@ -119,9 +119,7 @@ module YAVDB
                                    published_date
                                  end
 
-            vuln_id_stamp = (sidebar_data[:cve] && sidebar_data[:cve][0]) ||
-                            sidebar_data[:id].split(%r{-|:}).last ||
-                            disclosed_date
+            vuln_id_stamp = sidebar_data[:id].split(%r{-|:}).last || disclosed_date
             vuln_id       = "snykio:#{package_manager}:#{affected_package}:#{vuln_id_stamp}"
 
             YAVDB::Advisory.new(

data/lib/yavdb/utils/http.rb

@@ -47,6 +47,8 @@ module YAVDB
             case response
               when Net::HTTPNotFound then
                 raise ArgumentError, 'page not found'
+              when Net::HTTPTooManyRequests then
+                raise ArgumentError, 'too many requests'
               else
                 response.body.lines
             end

data/lib/yavdb/version.rb

@@ -16,6 +16,6 @@
 
 module YAVDB
 
-  VERSION = '0.4.4'
+  VERSION = '0.4.5'
 
 end

data/yavdb.gemspec

@@ -36,9 +36,11 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rubocop-rspec', ['~> 1.29']
 
   # Runtime
+  spec.add_runtime_dependency 'execjs', ['~> 2.7.0']
   spec.add_runtime_dependency 'json', ['~> 2.1']
   spec.add_runtime_dependency 'kramdown', ['~> 1.17']
   spec.add_runtime_dependency 'oga', ['~> 2.15']
   spec.add_runtime_dependency 'semantic_interval', ['~> 0.1']
+  spec.add_runtime_dependency 'therubyracer', ['~> 0.12']
   spec.add_runtime_dependency 'thor', ['~> 0.20']
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: yavdb
 version: !ruby/object:Gem::Version
-  version: 0.4.4
+  version: 0.4.5
 platform: ruby
 authors:
 - Rodrigo Fernandes
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-18 00:00:00.000000000 Z
+date: 2018-11-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -122,6 +122,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.29'
+- !ruby/object:Gem::Dependency
+  name: execjs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.7.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.7.0
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
@@ -178,6 +192,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: therubyracer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.12'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.12'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -227,6 +255,7 @@ files:
 - lib/yavdb/dtos/advisory.rb
 - lib/yavdb/source_types/git_repo.rb
 - lib/yavdb/sources/friends_of_php.rb
+- lib/yavdb/sources/npmjs.rb
 - lib/yavdb/sources/ossindex.rb
 - lib/yavdb/sources/ruby_advisory.rb
 - lib/yavdb/sources/snyk_io.rb