yavdb 0.1.0.pre.alpha.2

data/lib/yavdb/utils/cache.rb

@@ -0,0 +1,96 @@
+# yavdb - The Free and Open Source vulnerability database
+# Copyright (C) 2017-present Rodrigo Fernandes
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'json'
+require 'digest'
+require 'fileutils'
+require 'securerandom'
+
+module YAVDB
+  module Utils
+    module Cache
+
+      CACHE_MAIN_KEY = 'general'
+
+      def self.cache_contents(group_key, key, &body)
+        get(group_key, key) || put(group_key, key, body.call)
+      end
+
+      def self.cache_path(group_key, key, &body)
+        key_path = generate_cache_identifier(group_key, key)
+        body.call(key_path) unless File.exist?(key_path)
+        key_path
+      end
+
+      class << self
+
+        private
+
+        def get(group_key, key)
+          key_path = generate_cache_identifier(group_key, key)
+          File.open(key_path, 'rb') { |file| Marshal.load(file.read) } if File.exist?(key_path)
+        end
+
+        def put(group_key, key, value)
+          key_path = generate_cache_identifier(group_key, key)
+          File.open(key_path, 'wb') { |file| file.write(Marshal.dump(value)) }
+          value
+        end
+
+        def generate_cache_identifier(group_key, key)
+          group_key = sanitize_key(group_key)
+          key       = sanitize_key(key)
+
+          group_path = if group_key
+                         File.expand_path(File.join(YAVDB::Constants::DEFAULT_CACHE_PATH, group_key))
+                       else
+                         File.expand_path(File.join(YAVDB::Constants::DEFAULT_CACHE_PATH, CACHE_MAIN_KEY))
+                       end
+
+          FileUtils.mkdir_p(group_path) unless File.exist?(group_path)
+
+          File.expand_path(File.join(group_path, key))
+        end
+
+        def sanitize_key(key)
+          sanitized_key = key
+                            .gsub(%r{[^[:alnum:]]}, '-')
+                            .gsub(%r{(-)+}, '-')
+                            .gsub(%r{^-}, '')
+                            .gsub(%r{-$}, '')
+
+          if sanitized_key == '-'
+            hex_key = Digest::SHA256.hexdigest(key)
+
+            puts "Could not sanitize key(#{key}) using #{hex_key} instead"
+
+            hex_key
+          elsif sanitized_key.empty?
+            random_string = SecureRandom.hex
+
+            puts "Could not sanitize key(#{key}) using #{random_string} instead"
+
+            random_string
+          else
+            sanitized_key
+          end
+        end
+
+      end
+
+    end
+  end
+end

data/lib/yavdb/utils/exec.rb

@@ -0,0 +1,36 @@
+# yavdb - The Free and Open Source vulnerability database
+# Copyright (C) 2017-present Rodrigo Fernandes
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'English'
+
+require_relative '../constants'
+
+module YAVDB
+  module Utils
+    module Executor
+
+      def self.run(cmd)
+        puts "[Running] #{cmd}" if Constants::DEBUG
+        output = `#{cmd}`
+        {
+          :output   => output,
+          :success? => $CHILD_STATUS.success?
+        }
+      end
+
+    end
+  end
+end

data/lib/yavdb/utils/git.rb

@@ -0,0 +1,61 @@
+# yavdb - The Free and Open Source vulnerability database
+# Copyright (C) 2017-present Rodrigo Fernandes
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'tmpdir'
+
+require_relative '../constants'
+require_relative '../utils/exec'
+require_relative '../utils/cache'
+
+module YAVDB
+  module Utils
+    module Git
+
+      def self.get_contents(repo_url, repo_branch, with_cache = true, group_cache_key = 'git')
+        puts "Requesting #{repo_url}" if Constants::DEBUG
+
+        if with_cache
+          YAVDB::Utils::Cache.cache_path(group_cache_key, repo_url) do |repo_path|
+            download_or_update(repo_path, repo_url, repo_branch)
+            repo_path
+          end
+        else
+          repo_path = Dir.mktmpdir(group_cache_key)
+          download_or_update(repo_path, repo_url, repo_branch)
+          repo_path
+        end
+      end
+
+      def self.download_or_update(repo_path, repo_url, repo_branch, force_update = true)
+        puts "Downloading #{repo_url}" if Constants::DEBUG
+
+        if File.exist?(repo_path) && Dir.entries(repo_path) != ['.', '..']
+          if File.directory?(File.expand_path(File.join(repo_path, '.git')))
+            Dir.chdir(repo_path) do
+              YAVDB::Utils::Executor.run("git fetch --all && git reset --hard origin/#{repo_branch}") if force_update
+            end
+          else
+            puts "Repository directory already exists and is not a valid git repository in #{repo_path}"
+            exit(1)
+          end
+        else
+          YAVDB::Utils::Executor.run("git clone #{repo_url} -b #{repo_branch} #{repo_path}")
+        end
+      end
+
+    end
+  end
+end

data/lib/yavdb/utils/http.rb

@@ -0,0 +1,56 @@
+# yavdb - The Free and Open Source vulnerability database
+# Copyright (C) 2017-present Rodrigo Fernandes
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'net/https'
+
+require_relative '../utils/cache'
+
+module YAVDB
+  module Utils
+    module HTTP
+
+      def self.get_page_contents(url, with_cache = true, group_cache_key = 'http')
+        puts "Requesting #{url}"
+
+        if with_cache
+          YAVDB::Utils::Cache.cache_contents(group_cache_key, url) { do_request(url) }
+        else
+          do_request(url)
+        end
+      end
+
+      class << self
+
+        private
+
+        def do_request(url)
+          puts "Fetching #{url}"
+
+          url      = URI.parse(url)
+          response = Net::HTTP.get_response(url)
+          case response
+            when Net::HTTPNotFound then
+              raise ArgumentError, 'page not found'
+            else
+              response.body.lines
+          end
+        end
+
+      end
+
+    end
+  end
+end

data/lib/yavdb/utils/semver.rb

@@ -0,0 +1,79 @@
+# yavdb - The Free and Open Source vulnerability database
+# Copyright (C) 2017-present Rodrigo Fernandes
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+module YAVDB
+  module Utils
+    module SemVer
+
+      SEMANTIC_INTERVAL_REGEX = %r{([(\[].+?[)\]])}
+
+      def self.clean_versions(versions)
+        return if versions.nil? || (!versions.is_a?(String) && !versions.is_a?(Array))
+
+        versions = to_array(versions).map do |version|
+          if semantic_interval?(version)
+            convert_to_semver(version)
+          else
+            split_versions(version)
+          end
+        end
+
+        versions
+          .flatten
+          .map(&:strip)
+          .select { |str| str != '-' && !str.empty? }
+      end
+
+      class << self
+
+        private
+
+        def to_array(versions)
+          versions = [versions] if versions.is_a?(String)
+
+          versions
+            .flatten
+            .reject { |str| str.strip.empty? }
+        end
+
+        def semantic_interval?(version)
+          version =~ SEMANTIC_INTERVAL_REGEX
+        end
+
+        def convert_to_semver(version)
+          ver_tmp = version
+                      .scan(SEMANTIC_INTERVAL_REGEX)
+                      .flatten
+                      .map { |v| SemanticInterval.parse(v) }
+
+          if ver_tmp.any?
+            ver_tmp
+          else
+            version
+          end
+        end
+
+        def split_versions(version)
+          version
+            .strip
+            .split(',')
+        end
+
+      end
+
+    end
+  end
+end

data/lib/yavdb/utils/zip.rb

@@ -0,0 +1,64 @@
+# yavdb - The Free and Open Source vulnerability database
+# Copyright (C) 2017-present Rodrigo Fernandes
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'fileutils'
+
+require_relative '../constants'
+require_relative '../utils/cache'
+require_relative '../utils/exec'
+
+module YAVDB
+  module Utils
+    module Zip
+
+      def self.get_contents(zip_url, with_cache = true, group_cache_key = 'zip')
+        puts "Requesting #{zip_url}" if Constants::DEBUG
+
+        if with_cache
+          YAVDB::Utils::Cache.cache_path(group_cache_key, zip_url) do |zip_path|
+            do_request(zip_path, zip_url)
+            zip_path
+          end
+        else
+          zip_path = Dir.mktmpdir(group_cache_key)
+          do_request(zip_path, zip_url)
+          zip_path
+        end
+      end
+
+      class << self
+
+        private
+
+        def do_request(zip_path, zip_url)
+          puts "Downloading #{zip_url}" if Constants::DEBUG
+
+          resource_path = "#{zip_path}/resource.zip"
+
+          FileUtils.rm_rf(zip_path)
+          FileUtils.mkdir_p(zip_path)
+
+          YAVDB::Utils::Executor.run %(wget -O #{resource_path} #{zip_url})
+          YAVDB::Utils::Executor.run %(unzip #{resource_path} -d #{zip_path})
+
+          FileUtils.rm(resource_path)
+        end
+
+      end
+
+    end
+  end
+end

data/lib/yavdb/version.rb

@@ -0,0 +1,21 @@
+# yavdb - The Free and Open Source vulnerability database
+# Copyright (C) 2017-present Rodrigo Fernandes
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+module YAVDB
+
+  VERSION = '0.1.0-alpha.2'
+
+end

data/yavdb.gemspec

@@ -0,0 +1,44 @@
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+require 'yavdb/version'
+
+Gem::Specification.new do |spec|
+  spec.name = 'yavdb'
+  spec.version = YAVDB::VERSION
+  spec.authors = ['Rodrigo Fernandes']
+  spec.email = ['rodrigo.fernandes@tecnico.ulisboa.pt']
+  spec.summary = 'The Free and Open Source vulnerability database.'
+  spec.description = '
+    Yet Another Vulnerability Database
+    The Free and Open Source vulnerability database.
+  '
+  spec.homepage = 'https://github.com/rtfpessoa/yavdb'
+  spec.license = 'AGPL-3.0+'
+
+  spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features|database)/}) }
+  spec.bindir = 'bin'
+  spec.executables = ['yavdb', 'vulndb', 'vulnerabilitydb']
+  spec.require_paths = ['lib']
+
+  spec.required_ruby_version = '>= 2.3.7'
+
+  # Development
+  spec.add_development_dependency 'bundler', ['~> 1.16']
+  spec.add_development_dependency 'codacy-coverage'
+  spec.add_development_dependency 'rake', ['~> 12.3']
+  spec.add_development_dependency 'rspec', ['~> 3.8']
+  spec.add_development_dependency 'rspec_junit_formatter', ['~> 0.4']
+  spec.add_development_dependency 'simplecov'
+
+  # Linters
+  spec.add_development_dependency 'rubocop', ['~> 0.58']
+  spec.add_development_dependency 'rubocop-rspec', ['~> 1.27']
+
+  # Runtime
+  spec.add_runtime_dependency 'json', ['~> 2.1']
+  spec.add_runtime_dependency 'kramdown', ['~> 1.17']
+  spec.add_runtime_dependency 'oga', ['~> 2.15']
+  spec.add_runtime_dependency 'semantic_interval', ['~> 0.1']
+  spec.add_runtime_dependency 'thor', ['~> 0.20']
+end