fidius-cvedb 0.0.2

data/lib/db/migrate/20101122175402_create_nvd_entries.rb

@@ -0,0 +1,18 @@
+class CreateNvdEntries < ActiveRecord::Migration
+  def self.up
+    create_table :nvd_entries do |t|
+      t.string :cve
+      t.string :cwe
+      t.string :summary
+      t.datetime :published
+      t.datetime :last_modified
+
+      t.timestamps
+    end
+    add_index :nvd_entries, :cve
+  end
+
+  def self.down
+    drop_table :nvd_entries
+  end
+end

data/lib/db/migrate/20101125140254_create_vulnerability_references.rb

@@ -0,0 +1,16 @@
+class CreateVulnerabilityReferences < ActiveRecord::Migration
+  def self.up
+    create_table :vulnerability_references do |t|
+      t.string :name
+      t.string :link
+      t.string :source
+      t.integer :nvd_entry_id
+
+      t.timestamps
+    end
+  end
+
+  def self.down
+    drop_table :vulnerability_references
+  end
+end

data/lib/db/migrate/20101202100411_create_xmls.rb

@@ -0,0 +1,14 @@
+class CreateXmls < ActiveRecord::Migration
+  def self.up
+    create_table :xmls do |t|
+      t.string :name
+      t.datetime :create_time
+
+      t.timestamps
+    end
+  end
+
+  def self.down
+    drop_table :xmls
+  end
+end

data/lib/db/migrate/20101210141850_create_mscves.rb

@@ -0,0 +1,14 @@
+class CreateMscves < ActiveRecord::Migration
+  def self.up
+    create_table :mscves do |t|
+      t.integer :nvd_entry_id
+      t.string :name
+
+      t.timestamps
+    end
+  end
+
+  def self.down
+    drop_table :mscves
+  end
+end

data/lib/db/migrate/20110118124541_change_impacts_structure.rb

@@ -0,0 +1,13 @@
+class ChangeImpactsStructure < ActiveRecord::Migration
+  def self.up
+    add_column :cvsses, :availability_impact_id, :integer
+    add_column :cvsses, :confidentiality_impact_id, :integer
+    add_column :cvsses, :integrity_impact_id, :integer
+  end
+
+  def self.down
+    remove_column :cvsses, :availability_impact_id
+    remove_column :cvsses, :confidentiality_impact_id
+    remove_column :cvsses, :integrity_impact_id
+  end
+end

data/lib/db/migrate/20110118131643_destroy_vulnerable_configurations.rb

@@ -0,0 +1,13 @@
+class DestroyVulnerableConfigurations < ActiveRecord::Migration
+  def self.up
+    drop_table :vulnerable_configurations
+  end
+
+  def self.down
+    create_table :vulnerable_configurations do |t|
+      t.integer :nvd_entry_id
+      t.integer :product_id
+      t.timestamps
+    end
+  end
+end

data/lib/fidius-cvedb.rb

@@ -0,0 +1,17 @@
+require 'fidius-cvedb/version'
+
+module FIDIUS
+  module CveDb
+    GEM_BASE = File.join(ENV['GEM_HOME'], 'gems', "fidius-cvedb-#{VERSION}", 'lib')
+    RAILS_VERSION = Rails.version.to_i
+    
+    # If the used Rails version is 3 or beyond we use railties to load the rake
+    # tasks. Otherwise they are symlinked.
+    require 'fidius-cvedb/railtie' unless RAILS_VERSION < 3
+    
+    require (File.join GEM_BASE, 'models', 'fidius', 'cve_db', 'cve_connection.rb')
+    Dir.glob(File.join GEM_BASE, 'models', 'fidius', 'cve_db', '*.rb') do |rb|
+      require rb
+    end
+  end
+end

data/lib/fidius-cvedb/railtie.rb

@@ -0,0 +1,14 @@
+require 'fidius-cvedb'
+require 'rails'
+
+module FIDIUS
+  module CveDb
+    class Railtie < Rails::Railtie
+      rake_tasks do
+        load "tasks/parse_cves.rake"
+        load "tasks/db_backup.rake"
+        load "tasks/nvd_migrate.rake"
+      end       
+    end
+  end
+end

data/lib/fidius-cvedb/version.rb

@@ -0,0 +1,5 @@
+module FIDIUS
+  module CveDb
+    VERSION = "0.0.2"
+  end
+end

data/lib/models/fidius/cve_db/cve_connection.rb

@@ -0,0 +1,7 @@
+class FIDIUS::CveDb::CveConnection < ActiveRecord::Base
+  # If the application is using the cve_db as foreign database this should be
+  # uncommented.
+  database_yml = YAML.load_file('config/database.yml')
+  establish_connection database_yml['cve_db']
+  self.abstract_class = true
+end

data/lib/models/fidius/cve_db/cvss.rb

@@ -0,0 +1,5 @@
+class FIDIUS::CveDb::Cvss < FIDIUS::CveDb::CveConnection
+  has_one :confidentiality_impact
+  has_one :availability_impact
+  has_one :integrity_impact
+end

data/lib/models/fidius/cve_db/impact.rb

@@ -0,0 +1,2 @@
+class FIDIUS::CveDb::Impact < FIDIUS::CveDb::CveConnection
+end

data/lib/models/fidius/cve_db/mscve.rb

@@ -0,0 +1,3 @@
+class FIDIUS::CveDb::Mscve < FIDIUS::CveDb::CveConnection
+  belongs_to :nvd_entry
+end

data/lib/models/fidius/cve_db/nvd_entry.rb

@@ -0,0 +1,23 @@
+class FIDIUS::CveDb::NvdEntry < FIDIUS::CveDb::CveConnection
+  
+  has_one :cvss
+  has_one :mscve
+  
+  has_many :vulnerable_softwares
+  has_many :vulnerable_configurations
+  has_many :vulnerability_references
+
+  validates_uniqueness_of :cve
+  
+  def references_string
+    res = ""
+    vulnerability_references.each_with_index do |reference, i|
+      # http://www.foo-bar.de/sub/sub-sub -> www.foo-bar.de
+      link_name = reference.link.scan(/(?:https?|s?ftp):\/\/([^\/]+)/).to_s
+      res += "<a href=\"#{reference.link}\">#{link_name}</a>"
+      res += " | " unless i == vulnerability_references.size-1
+    end
+    res
+  end
+  
+end

data/lib/models/fidius/cve_db/product.rb

@@ -0,0 +1,6 @@
+class FIDIUS::CveDb::Product < FIDIUS::CveDb::CveConnection
+  
+  has_many :vulnerable_softwares
+  has_many :nvd_entries, :through => :vulnerable_softwares
+  
+end

data/lib/models/fidius/cve_db/vulnerability_reference.rb

@@ -0,0 +1,3 @@
+class FIDIUS::CveDb::VulnerabilityReference < FIDIUS::CveDb::CveConnection
+  belongs_to :nvd_entry
+end

data/lib/models/fidius/cve_db/vulnerable_configuration.rb

@@ -0,0 +1,6 @@
+class FIDIUS::CveDb::VulnerableConfiguration < FIDIUS::CveDb::CveConnection
+  
+  belongs_to :nvd_entry
+  belongs_to :product
+  
+end

data/lib/models/fidius/cve_db/vulnerable_software.rb

@@ -0,0 +1,6 @@
+class FIDIUS::CveDb::VulnerableSoftware < FIDIUS::CveDb::CveConnection
+  
+  belongs_to :nvd_entry
+  belongs_to :product
+  
+end

data/lib/models/fidius/cve_db/xml.rb

@@ -0,0 +1,2 @@
+class FIDIUS::CveDb::Xml < FIDIUS::CveDb::CveConnection
+end

data/lib/tasks/db_backup.rake

@@ -0,0 +1,30 @@
+require 'find'
+
+BACKUP_DIR = "db"
+RAILS_ENV = "development" # TODO improve :D
+MAX_BACKUPS = 5
+
+# Source: http://blog.craigambrose.com/articles/2007/03/01/a-rake-task-for-database-backups
+namespace :db do  
+  desc "Backup the database to a file." 
+  task :backup => [:environment] do
+    datestamp = Time.now.strftime("%Y-%m-%d_%H-%M-%S")
+    base_path = BACKUP_DIR || "db" 
+    backup_base = File.join(base_path, 'backup')
+    backup_folder = File.join(backup_base, datestamp)
+    backup_file = File.join(backup_folder, "#{RAILS_ENV}_dump.sql.gz")
+    FileUtils.mkdir_p(backup_folder)
+    db_config = ActiveRecord::Base.configurations[RAILS_ENV]
+    sh "mysqldump -u #{db_config['username']} -p#{db_config['password']} -Q --add-drop-database --opt #{db_config['database']} | gzip -c > #{backup_file}"
+    dir = Dir.new(backup_base)
+    all_backups = dir.entries[2..-1].sort.reverse
+    puts "Created backup: #{backup_file}"
+    max_backups = MAX_BACKUPS || 20
+    unwanted_backups = all_backups[max_backups..-1] || []
+    for unwanted_backup in unwanted_backups
+      FileUtils.rm_rf(File.join(backup_base, unwanted_backup))
+      puts "deleted #{unwanted_backup}" 
+    end
+    puts "Deleted #{unwanted_backups.length} backups, #{all_backups.length - unwanted_backups.length} backups available" 
+  end
+end

data/lib/tasks/nvd_migrate.rake

@@ -0,0 +1,25 @@
+require 'yaml'
+
+namespace :nvd do 
+  desc 'Execute NVD migrations'
+  task :migrate do
+    db_connect
+    ActiveRecord::Migrator.migrate(File.join FIDIUS::CveDb::GEM_BASE, 'db', 'migrate')
+  end
+  
+  desc "Drop NVD database"
+  task :drop do
+    #db_connect
+    raise NotImplementedError
+    
+  end
+end
+
+def db_connect
+  db_config = YAML.load_file(File.join(Rails.root.to_s, 'config', 'database.yml'))
+    if db_config["cve_db"]
+      ActiveRecord::Base::establish_connection db_config["cve_db"]
+    else
+      puts "No configuration found for cve_db in config/database.yml!"
+    end
+end

data/lib/tasks/parse_cves.rake

@@ -0,0 +1,146 @@
+require 'open-uri'
+require 'net/http'
+require 'nokogiri'
+require 'fidius-cvedb'
+
+BASE_URL = "http://static.nvd.nist.gov/feeds/xml/cve/"
+DOWNLOAD_URL = "http://nvd.nist.gov/download.cfm"
+#GEM_BASE = File.join(ENV['GEM_HOME'], 'gems', "fidius-cvedb-#{FIDIUS::CveDb::VERSION}", 'lib')
+XML_DIR = File.join(Dir.pwd, "cveparser", "xml")
+ANNUALLY_XML = /nvdcve-2[.]0-\d{4}[.]xml/
+
+# modified xml includes all recent published and modified cve entries
+MODIFIED_XML = "nvdcve-2.0-modified.xml"
+
+namespace :nvd do 
+  desc 'Parses local XML-File.'
+  task :parse, :file_name do |t,args|
+    cve_main '-p', args[:file_name]
+  end
+  
+  desc 'Downloads XML-File from NVD. (with names from nvd:list_remote)'
+  task :get, :xml_name do |t,args|
+    if args[:xml_name]
+      wget args[:xml_name]
+    else
+      puts "Please call task with 'rake nvd:get[<xml_name>]'!"
+    end 
+  end
+  
+  desc 'Lists the locally available NVD-XML-Datafeeds.'
+  task :list_local do
+    puts local_xmls
+  end  
+
+  desc 'Lists the remotely available NVD-XML-Datafeeds.'
+  task :list_remote do
+    xmls = remote_xmls
+    if xmls
+      puts "#{xmls.size} XMLs available:\n------"
+      puts xmls
+    else
+      puts "No suitable XMLs found."
+    end
+  end
+
+  desc "Downloads the modified.xml from nvd.org and stores it's content in the database."
+  task :update do
+    wget MODIFIED_XML
+    cve_main '-u', MODIFIED_XML
+  end
+
+  desc "Initializes the CVE-DB, parses all annual CVE-XMLs and removes duplicates."
+  task :initialize do
+    init
+  end
+
+  desc "Creates the mapping between CVEs and Microsoft Security Bulletin Notation in the database."
+  task :mscve do
+    cve_main '-m'
+  end
+end
+
+# ---------- Helper - Methods ---------- #
+
+# Initializes the CVE-DB with all CVE data available in the NVD.
+def init
+  local_x = local_xmls
+  if local_x
+    puts "WARNING: The XML directory already contains XML files. "+
+      "nvd:initialize is intended to be used only once to set up the "+
+      "CVE-Entries. To update the CVE-Entries use nvd:update.\n\n"+
+      "If you don't cancel the task I'll proceed in 20 seconds."
+    sleep 20
+  end
+  puts "[*] Looking for XMLs at #{DOWNLOAD_URL}"
+  remote_x = remote_xmls
+  r_ann_xmls = []
+  remote_x.each do |xml|
+    r_ann_xmls << xml if xml.match ANNUALLY_XML
+  end
+  puts "[*] I've found #{r_ann_xmls.size} annually XML files remotely."
+  puts "[*] Checking locally available XMLs."
+  l_ann_xmls = []
+  if local_x
+    local_x.each do |xml|
+      l_ann_xmls << xml if xml.match ANNUALLY_XML
+    end
+  end
+  puts "[*] I've found #{l_ann_xmls.size} annually XML files locally. I'll "+
+    "download the missing XMLs now."
+  r_ann_xmls.each do |xml|
+    wget xml unless l_ann_xmls.include? xml
+    puts "Downloaded #{xml}."
+  end
+  puts "[*] All available files downloaded, parsing the XMLs now."
+  l_ann_xmls.each do |xml|
+    cve_main '-p', xml
+  end
+  puts "[*] All local XMLs parsed."
+  cve_main '-f'
+  puts "[*] Initializing done."
+end
+
+# Returns an array of xmls that were previously downloaded
+def local_xmls
+  if Dir.exists?(XML_DIR)
+    entries = []
+    dir = Dir.new XML_DIR
+    dir.each do |entry|
+      entries << entry if entry =~ /.+\.xml$/
+    end
+    return (entries.empty? ? nil : entries)
+  else
+    puts "ERROR: There is no directory \"#{XML_DIR}\" where I can look for XMLs."
+    return nil
+  end
+end
+
+# Returns an array of available xmls or nil if none are found.
+def remote_xmls
+  doc = Nokogiri::HTML(open(DOWNLOAD_URL))
+  links = doc.css("div.rightbar > a")
+  xmls = []
+  links.each do |link|
+    link_name = link.attributes['href'].to_s
+    if link_name
+      xmls << link_name.split("/").last if link_name.include? BASE_URL
+    end    
+  end
+  xmls.empty? ? nil : xmls
+end
+
+# Calls the main.rb script with appropriate options
+def cve_main option, file = ''
+  runner_version = Rails.version[0].to_i < 3 ? "ruby script/runner" : "rails runner"
+  main_script = "#{runner_version} #{File.join(FIDIUS::CveDb::GEM_BASE, "cveparser", "main.rb")}" 
+  param = file.empty? ? file : " #{File.join(XML_DIR, file)}"
+  puts "Working Dir : #{Dir.pwd}"
+  sh "#{main_script} #{option} #{param}" 
+end
+
+# Simple wget 
+def wget file
+  FileUtils.mkdir_p(XML_DIR)
+  sh "wget -O#{File.join(XML_DIR, file)} #{BASE_URL + file}"
+end

data/test/cve_parser_test.rb

@@ -0,0 +1,25 @@
+$LOAD_PATH.unshift File.expand_path("../lib/cveparser/")
+require 'parser'
+require 'test/unit'
+
+class TestCveParser < Test::Unit::TestCase
+  
+  include FIDIUS::NVDParser
+  
+  def test_should_parse_2_0_only
+    assert_raise(RuntimeError) { FIDIUS::NVDParser.parse_cve_file 'test_v2.xml' }
+  end
+  
+  def test_should_find_1_reference
+    entries = FIDIUS::NVDParser.parse_cve_file 'test_references.xml'
+    assert_equal 1, entries.first.references.size, "The test_references.xml " +
+        "contains one reference which should be found."
+  end
+  
+  def test_should_find_3_nvd_entries
+    entries = FIDIUS::NVDParser.parse_cve_file 'test_entries.xml'
+    assert_equal 3, entries.size, "The test_entries.xml contains 3 NVD " +
+        "entries which should be returned in an array."
+  end
+  
+end