yavdb 0.1.0.pre.alpha.2

data/lib/yavdb/dtos/advisory.rb

@@ -0,0 +1,102 @@
+# yavdb - The Free and Open Source vulnerability database
+# Copyright (C) 2017-present Rodrigo Fernandes
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+module YAVDB
+  # TODO: Enable `Style/StructInheritance` - check `attr_reader:` or `initialize` method
+  class Advisory <
+
+    Struct.new(
+      :id, # [String]
+      :title, # [String]
+      :description, # [String]
+      :affected_package, # [String]
+      :vulnerable_versions, # [Array<String>] (Optional)
+      :unaffected_versions, # [Array<String>] (Optional)
+      :patched_versions, # [Array<String>] (Optional)
+      :severity, # [String] (Optional)
+      :package_manager, # [String]
+      :cve, # [Array<String>] (Optional)
+      :cwe, # [Array<String>] (Optional)
+      :osvdb, # [String] (Optional)
+      :cvss_v2_vector, # [String] (Optional)
+      :cvss_v2_score, # [String] (Optional)
+      :cvss_v3_vector, # [String] (Optional)
+      :cvss_v3_score, # [String] (Optional)
+      :disclosed_date, # [Date]
+      :created_date, # [Date]
+      :last_modified_date, # [Date]
+      :credit, # [Array<String>]
+      :references, # [Array<String>]
+      :source_url # [String]
+    )
+
+    def self.load(path)
+      data = YAML.load_file(path)
+
+      raise("Advisory data in #{path.dump} was not an Array") unless data.is_a?(Array)
+
+      data.map do |advisory|
+        raise("Advisory data in #{path.dump} was not a Hash") unless advisory.is_a?(Hash)
+
+        new(
+          advisory['id'],
+          advisory['title'],
+          advisory['description'],
+          advisory['affected_package'],
+          advisory['vulnerable_versions'],
+          advisory['unaffected_versions'],
+          advisory['patched_versions'],
+          advisory['severity'],
+          advisory['package_manager'],
+          advisory['cve'],
+          advisory['cwe'],
+          advisory['osvdb'],
+          advisory['cvss_v2_vector'],
+          advisory['cvss_v2_score'],
+          advisory['cvss_v3_vector'],
+          advisory['cvss_v3_score'],
+          advisory['disclosed_date'],
+          advisory['created_date'],
+          advisory['last_modified_date'],
+          advisory['credit'],
+          advisory['references'],
+          advisory['source_url']
+        )
+      end
+    end
+
+    def to_map
+      map = {}
+      members.each do |m|
+        next unless self[m] && (
+        (self[m].is_a?(String) && !self[m].empty?) ||
+          (self[m].is_a?(Array) && self[m].any?))
+
+        map[m.to_s] = self[m] if self[m]
+      end
+      map
+    end
+
+    def to_json(*args)
+      to_map.to_json(*args)
+    end
+
+    def to_yaml(*args)
+      to_map.to_yaml(*args)
+    end
+
+  end
+end

data/lib/yavdb/source_types/git_repo.rb

@@ -0,0 +1,35 @@
+# yavdb - The Free and Open Source vulnerability database
+# Copyright (C) 2017-present Rodrigo Fernandes
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require_relative '../utils/git'
+
+module YAVDB
+  module SourceTypes
+    module GitRepo
+
+      def self.search(file_pattern, repo_url, repo_branch = 'master', with_cache = true)
+        repo_path = YAVDB::Utils::Git.get_contents(repo_url, repo_branch, with_cache)
+
+        file_paths = Dir.chdir(repo_path) do
+          Dir.glob(file_pattern).select { |f| File.file?(f) }
+        end
+
+        Hash[repo_path => file_paths]
+      end
+
+    end
+  end
+end

data/lib/yavdb/sources/friends_of_php.rb

@@ -0,0 +1,87 @@
+# yavdb - The Free and Open Source vulnerability database
+# Copyright (C) 2017-present Rodrigo Fernandes
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'date'
+require 'yaml'
+
+require_relative '../dtos/advisory'
+require_relative '../source_types/git_repo'
+
+module YAVDB
+  module Sources
+    module FriendsOfPHP
+      class Client
+
+        REPOSITORY_URLS = [
+          'https://github.com/FriendsOfPHP/security-advisories',
+          'https://github.com/Cotya/magento-security-advisories'
+        ].freeze
+
+        PACKAGE_MANAGER = 'packagist'.freeze
+
+        def self.advisories
+          REPOSITORY_URLS.map do |url|
+            YAVDB::SourceTypes::GitRepo.search('*/*/*.yaml', url).map do |repo_path, file_paths|
+              Dir.chdir(repo_path) do
+                file_paths.map do |file_path|
+                  advisory_hash = YAML.load_file(file_path)
+                  url           = "#{url}/blob/master/#{file_path}"
+                  create(url, advisory_hash)
+                end
+              end
+            end
+          end.flatten
+        end
+
+        def self.create(url, advisory_hash)
+          date = Date.parse('1970-01-01')
+
+          versions = advisory_hash['branches'].map do |_, info|
+            date = Date.strptime(info['time'].to_s, '%Y-%m-%d %H:%M:%S %z') if info['time']
+            info['versions'].join(' ')
+          end.flatten
+
+          package_name = advisory_hash['reference'].gsub(%r{composer:\/\/(.*)}, '\1')
+          YAVDB::Advisory.new(
+            "friendsofphp:packagist:#{package_name}:#{date}",
+            advisory_hash['title'],
+            nil, #:description
+            package_name,
+            versions, #:vulnerable_versions
+            nil, #:unaffected_versions
+            nil, #:patched_versions
+            nil, #:severity
+            PACKAGE_MANAGER,
+            [advisory_hash['cve']].reject { |cve| cve == '~' },
+            nil, #:cwe
+            nil, #:osvdb
+            nil, #:cvss_v2_vector
+            nil, #:cvss_v2
+            nil, #:cvss_v3_vector
+            nil, #:cvss_v3
+            date,
+            date,
+            date,
+            ['FriendsOfPHP'],
+            [advisory_hash['link']],
+            url
+          )
+        end
+
+      end
+    end
+  end
+end

data/lib/yavdb/sources/nodesecurity_io.rb

@@ -0,0 +1,120 @@
+# yavdb - The Free and Open Source vulnerability database
+# Copyright (C) 2017-present Rodrigo Fernandes
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'oga'
+require 'oga/xml/entities'
+require 'date'
+
+require_relative '../dtos/advisory'
+require_relative '../utils/http'
+
+module YAVDB
+  module Sources
+    module NodeSecurityIO
+      class Client
+
+        API_URL     = 'https://api.nodesecurity.io/advisories'
+        WEBSITE_URL = 'https://nodesecurity.io/advisories'
+
+        def self.advisories
+          fetch_advisories.map do |advisory_hash|
+            create(advisory_hash)
+          end
+        end
+
+        class << self
+
+          private
+
+          def fetch_advisories
+            offset     = 0
+            advisories = []
+
+            loop do
+              nodesecurity    = YAVDB::Utils::HTTP.get_page_contents("#{API_URL}?offset=#{offset}", true, 'nodesecurity.io/advisories')
+              advisories_json = JSON.parse(nodesecurity.join)
+
+              advisories_json['count'].positive? ? advisories = advisories.concat(advisories_json['results']) : break
+
+              offset += advisories_json['count']
+            end
+
+            advisories
+          end
+
+          def create(advisory_hash)
+            publish_date = Date.parse(advisory_hash['publish_date'])
+            created_at   = Date.parse(advisory_hash['created_at'])
+            updated_at   = Date.parse(advisory_hash['updated_at'])
+
+            vulnerable_versions =
+              if advisory_hash['vulnerable_versions'].nil? || advisory_hash['vulnerable_versions'].empty?
+                '*'
+              else
+                advisory_hash['vulnerable_versions']
+              end
+
+            YAVDB::Advisory.new(
+              "nodesecurity:npm:#{advisory_hash['module_name']}:#{publish_date}",
+              advisory_hash['title'],
+              advisory_hash['overview'],
+              advisory_hash['module_name'],
+              [vulnerable_versions],
+              nil, #:unaffected_versions
+              advisory_hash['patched_versions'],
+              severity(advisory_hash['cvss_score']),
+              'npm',
+              advisory_hash['cves'],
+              nil, #:cwe
+              nil, #:osvdb
+              nil, #:cvss_v2_vector
+              nil, #:cvss_v2_score
+              advisory_hash['cvss_vector'],
+              advisory_hash['cvss_score'],
+              publish_date,
+              created_at,
+              updated_at,
+              [advisory_hash['author']],
+              clean_references(advisory_hash['references']),
+              "#{WEBSITE_URL}/#{advisory_hash['id']}"
+            )
+          end
+
+          def severity(cvss_score)
+            case cvss_score
+              when 0.0..3.3 then
+                'low'
+              when 3.3..6.6 then
+                'medium'
+              else
+                'high'
+            end
+          end
+
+          def clean_references(references)
+            if references
+              [references.gsub(%r{.*?(http.+)}, '\1')]
+            else
+              []
+            end
+          end
+
+        end
+
+      end
+    end
+  end
+end

data/lib/yavdb/sources/ossindex.rb

@@ -0,0 +1,137 @@
+# yavdb - The Free and Open Source vulnerability database
+# Copyright (C) 2017-present Rodrigo Fernandes
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'oga'
+require 'oga/xml/entities'
+
+require_relative '../dtos/advisory'
+require_relative '../utils/http'
+
+module YAVDB
+  module Sources
+    module OSSIndex
+      class Client
+
+        API_URL               = 'https://ossindex.net'
+        PACKAGE_MANAGERS      = ['npm', 'maven', 'composer', 'nuget', 'rubygems', 'pypi']
+        PACKAGE_MANAGER_ALIAS = Hash['composer' => 'packagist']
+
+        def self.advisories
+          PACKAGE_MANAGERS.map do |package_manager|
+            packages = fetch_packages(package_manager)
+            parse_vulnerabilities(package_manager, packages)
+          end.flatten
+        end
+
+        class << self
+
+          private
+
+          def fetch_packages(package_manager)
+            next_url = start_url(package_manager)
+            packages = []
+
+            while next_url
+              ossindex      = YAVDB::Utils::HTTP.get_page_contents(next_url, true, 'ossindex/advisories')
+              ossindex_json = JSON.parse(ossindex.join)
+              page_packages = ossindex_json['packages']
+
+              packages.concat(page_packages)
+
+              next_url = ossindex_json['next']
+            end
+
+            packages
+          end
+
+          def parse_vulnerabilities(package_manager, packages)
+            packages
+              .map do |package|
+              package['vulnerabilities'].map do |advisory|
+                create(package_manager, package, advisory)
+              end
+            end.flatten
+          end
+
+          def create(package_manager, package, advisory)
+            published_date = Date.strptime((advisory['published'] / 1000).to_s, '%s')
+            updated_date   = Date.strptime((advisory['updated'] / 1000).to_s, '%s')
+
+            cve = if advisory['cve']
+                    [advisory['cve']].map(&:strip).reject(&:empty?)
+                  else
+                    []
+                  end
+
+            package_manager = PACKAGE_MANAGER_ALIAS[package_manager] || package_manager
+
+            package_name =
+              if package_manager == 'maven'
+                "#{package['group']}:#{package['name']}"
+              elsif package_manager == 'packagist'
+                "#{package['group']}/#{package['name']}"
+              else
+                package['name']
+              end
+
+            versions = advisory['versions']
+                         .map { |v| v.split('||') }
+                         .flatten
+                         .map(&:strip)
+                         .reject(&:empty?)
+                         .reject { |v| v == '-' }
+            versions = ['*'] unless versions.any?
+
+            YAVDB::Advisory.new(
+              "ossindex:#{package_manager}:#{package_name}:#{published_date}",
+              advisory['title'],
+              advisory['description'],
+              package_name,
+              versions,
+              nil, #:unaffected_versions
+              nil, #:patched_versions
+              nil, #:severity
+              package_manager,
+              cve,
+              nil, #:cwe
+              nil, #:osvdb
+              nil, #:cvss_v2_vector
+              nil, #:cvss_v2_score
+              nil, #:cvss_v3_vector
+              nil, #:cvss_v3_score
+              published_date,
+              published_date,
+              updated_date,
+              ['OSSIndex'],
+              advisory['references'],
+              website_url(package['id'])
+            )
+          end
+
+          def start_url(package_manager)
+            "#{API_URL}/v2.0/vulnerability/pm/#{package_manager}/fromtill/0/-1"
+          end
+
+          def website_url(id)
+            "#{API_URL}/resource/package/#{id}/vulnerabilities"
+          end
+
+        end
+
+      end
+    end
+  end
+end