cve_crawler 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 9c5c9a85897586ed3cf7cc66e9e97b9b7b09269b
+  data.tar.gz: 14af3dafda87aef7f5697c5d0b080036a9dc2488
+SHA512:
+  metadata.gz: 272da0467bb4e39f284a810e6d304929cb12085cb57e838387e6d2042613f75541b1ba8af882d004ef7bfc7be0c377dde054732d373488cb8f5fc0dd3e1a891c
+  data.tar.gz: d319e15080382fec62bf9a36452b94b35c0f9969687fcd19a23e45c29639e28b83b8733defce524cb347cee7c7a9c78a405b8faae4b9f4dbe49b77e4b3a0d8b6

data/lib/cve_crawler/cve_core.rb

@@ -0,0 +1,42 @@
+require 'time'
+require 'cve_crawler'
+require 'cve_parser'
+
+module CVE
+  VERSION_MAJOR = 0
+  VERSION_MINOR = 1
+  VERSION_BUILD = 0
+  VERSION = "#{VERSION_MAJOR}.#{VERSION_MINOR}.#{VERSION_BUILD}".freeze
+
+  class Core
+    def initialize(crawl_type='default', verify_cert=true, user_agent=nil, filters=nil)
+      unless user_agent
+        user_agent = create_user_agent
+      end
+
+      @crawler = Crawler.new(crawl_type, verify_cert, user_agent)
+      @parser = Parser.new(filters)
+    end
+
+    def fetch
+      body = crawl.body
+      parse(body)
+    end
+
+    def crawl
+      @crawler.crawl
+    end
+
+    def parse(data)
+      @parser.parse(data)
+    end
+
+    def create_user_agent
+      'RubyCVECrawler/' + VERSION + ' (https://github.com/zarthus/ruby-cve-crawler)'
+    end
+
+    def inspect
+      "<CVE::Core crawler=#{@crawler.inspect} parser=#{@parser.inspect}>"
+    end
+  end
+end

data/lib/cve_crawler/cve_crawler.rb

@@ -0,0 +1,37 @@
+require 'net/https'
+require 'uri'
+
+module CVE
+  class Crawler
+    DATA_FEED_DEFAULT = URI('https://nvd.nist.gov/download/nvd-rss.xml')
+    DATA_FEED_ANALYZED = URI('https://nvd.nist.gov/download/nvd-rss-analyzed.xml')
+
+    def initialize(type, verify_cert, user_agent)
+      @crawl_url = type.downcase == 'analyzed' ? DATA_FEED_ANALYZED : DATA_FEED_DEFAULT
+      @verify_cert = verify_cert ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
+      @user_agent = user_agent
+    end
+
+    attr_reader :crawl_url, :verify_cert, :user_agent
+
+    def crawl
+      http = Net::HTTP.new(@crawl_url.host, @crawl_url.port)
+      http.use_ssl = @crawl_url.scheme == 'https'
+
+      if http.use_ssl?
+        http.verify_mode = @verify_cert
+      end
+
+      request = Net::HTTP::Get.new(@crawl_url, {'User-Agent' => @user_agent})
+      response = http.request(request)
+
+      response.value  # Raise an error if status is not 200
+
+      response
+    end
+
+    def inspect
+      "#<CVE::Crawler url=#{@crawl_url.to_s}>"
+    end
+  end
+end

data/lib/cve_crawler/cve_filter.rb

@@ -0,0 +1,35 @@
+module CVE
+  class Filter
+    def initialize(history_size=100)
+      @history = []
+      @history_size = history_size
+    end
+
+    attr_reader :history_size
+
+    def filter(contents)
+      return true unless contents.is_a?(Vulnerability)
+      return true if cve_exists?(contents.identifier)
+
+      history_check_limit
+      false
+    end
+
+    def cve_exists?(identifier)
+      return true if @history.include?(identifier)
+
+      @history << identifier
+      false
+    end
+
+    def history_check_limit
+      if @history.length >= @history_size
+        @history.drop((@history_size / 2).round)
+      end
+    end
+
+    def inspect
+      "#<CVE::Filter history=#{@history.count} limit=#{@history_size}>"
+    end
+  end
+end

data/lib/cve_crawler/cve_parser.rb

@@ -0,0 +1,74 @@
+require 'rss'
+require 'cve_filter'
+require 'cve_vulnerability'
+
+module CVE
+  class Parser
+    def initialize(filters=nil)
+      @filters = []
+
+      @filters << CVE::Filter.new
+      @filters << filters if filters
+    end
+
+    def parse(content)
+      results = parse_rss_feed(content)
+
+      filter(results)
+    end
+
+    def parse_rss_feed(content)
+      rss = RSS::Parser.parse(content)
+
+      parse_items(rss)
+    end
+
+    def parse_items(rss)
+      items = []
+
+      if rss.nil?
+        raise 'RSS object failed to parse, is it valid XML?'
+      end
+
+      rss.items.each do |item|
+        items << parse_item(item)
+      end
+
+      items
+    end
+
+    def parse_item(item)
+      CVE::Vulnerability.new({
+        :identifier => extract_cve_identifier(item.title),
+        :title => item.title,
+        :link => item.link,
+        :description => item.description,
+        :date => item.date
+      })
+    end
+
+    def extract_cve_identifier(title)
+      title.split(' ')[0]
+    end
+
+    def filter(results)
+      filtered_contents = []
+
+      results.each do |result|
+        result_ok = true
+
+        @filters.each do |filter|
+          result_ok = result_ok && !filter.filter(result)
+        end
+
+        filtered_contents << result if result_ok
+      end
+
+      filtered_contents
+    end
+
+    def inspect
+      "<CVE::Parser filters=#{@filters.inspect}>"
+    end
+  end
+end

data/lib/cve_crawler/cve_vulnerability.rb

@@ -0,0 +1,62 @@
+module CVE
+  class Vulnerability
+    SOFTWARE_EXTRACT_REGEXP = Regexp.new('[(, ]([^(), ]+)')
+
+    def initialize(data)
+      unless data.instance_of?(Hash)
+        raise 'CVE Vulnerability needs to be initialized with a hash'
+      end
+
+      if malformed?(data)
+        raise 'CVE Vulnerability data is malformed'
+      end
+
+      @identifier = data[:identifier]
+      @date = data[:date]
+      @description = data[:description]
+      @link = data[:link]
+      @title = data[:title]
+      @affected_software = extract_software_from_title(data[:title])
+    end
+
+    attr_reader :identifier, :date, :description, :link, :title, :affected_software
+
+    def malformed?(data)
+      !(data.has_key?(:identifier) && data.has_key?(:date) && data.has_key?(:description) &&
+          data.has_key?(:link) && data.has_key?(:title))
+    end
+
+    def extract_software_from_title(title)
+      software = []
+
+      title.scan(SOFTWARE_EXTRACT_REGEXP) do |scan|
+        software << scan[0]
+      end
+
+      software.count == 0 ? nil : software
+    end
+
+    def affected_count
+      @affected_software.nil? ? 0 : @affected_software.count
+    end
+
+    def equal?(cve_item, strict=false)
+      return false unless cve_item.is_a?(Vulnerability)
+
+      if strict
+        return @identifier == cve_item.identifier && @link == cve_item.link && @date.utc.iso8601 == cve_item.date.utc.iso8601 &&
+            @title == cve_item.title && @description == cve_item.description
+      end
+
+      @identifier == cve_item.identifier && @link == cve_item.link
+    end
+
+    def to_s
+      "#{@title} - #{@link}"
+    end
+
+    def inspect
+      "#<CVE::Vulnerability id=#{@identifier} affected=#{affected_count}>"
+    end
+  end
+end

data/spec/crawler_spec.rb

@@ -0,0 +1,15 @@
+require_relative 'spec_helper'
+
+describe CVE::Crawler do
+  it 'should crawl the analyzed file when constructed with the analyzed type' do
+    crawler = CVE::Crawler.new('analyzed', false, false)
+
+    expect(crawler.crawl_url).to equal(CVE::Crawler::DATA_FEED_ANALYZED)
+  end
+
+  it 'should crawl the default file when constructed any non-analyzed type' do
+    crawler = CVE::Crawler.new('anything', false, false)
+
+    expect(crawler.crawl_url).to equal(CVE::Crawler::DATA_FEED_DEFAULT)
+  end
+end

data/spec/filter_spec.rb

@@ -0,0 +1,19 @@
+require_relative 'spec_helper'
+
+describe CVE::Filter do
+  it 'should filter repeated identifiers' do
+    filter = CVE::Filter.new
+    vulnerability = VulnerabilityMock.generate
+
+    expect(filter.filter(vulnerability)).to equal(false)
+    expect(filter.filter(vulnerability)).to equal(true)
+  end
+
+  it 'should not filter different identifiers' do
+    filter = CVE::Filter.new
+
+    3.times do
+      expect(filter.filter(VulnerabilityMock.generate)).to equal(false)
+    end
+  end
+end

data/spec/parser_spec.rb

@@ -0,0 +1,38 @@
+require_relative 'spec_helper'
+
+describe CVE::Parser do
+  crawler_mock = CrawlerResultMock.new
+  parser = CVE::Parser.new
+
+  it 'should parse a full xml body with three items and return an array of CVE items' do
+    result = parser.parse(crawler_mock.xml_full)
+
+    expect(result).to be_a(Array)
+    expect(result.length).to eq(3)
+
+    all_cve_items = true
+    result.each do |value|
+      all_cve_items = all_cve_items && value.is_a?(CVE::Vulnerability)
+    end
+
+    expect(all_cve_items).to equal(true)
+  end
+
+  it 'should raise an error when non-xml is passed' do
+    expect{ parser.parse('Not xml') }.to raise_error(RuntimeError)
+  end
+
+  it 'should extract the CVE identifier from a title' do
+    title = VulnerabilityMock.new.title
+
+    expect(parser.extract_cve_identifier(title)).to match(/^CVE\-\d{4}\-\d+$/)
+  end
+
+  it 'should accept a custom filter' do
+    CVE::Parser.new(CVE::Filter.new)
+  end
+
+  it 'should accept an array of custom filters' do
+    CVE::Parser.new([CVE::Filter.new, CVE::Filter.new])
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,84 @@
+require 'rspec'
+require_relative File.join('..', 'lib', 'cve_crawler', 'cve_core')
+
+class CrawlerResultMock
+  def xml_full
+    <<-eos
+<?xml version="1.0" encoding="UTF-8"?>
+<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns="http://purl.org/rss/1.0/" xmlns:dc="http://purl.org/dc/elements/1.1/">
+  <channel rdf:about="http://web.nvd.nist.gov/view/vuln/search">
+    <title>National Vulnerability Database</title>
+    <link>http://web.nvd.nist.gov/view/vuln/search</link>
+    <description>This feed contains the most recent CVE cyber vulnerabilities published within the National Vulnerability Database.</description>
+    <items>
+      <rdf:Seq>
+        <rdf:li rdf:resource="http://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2015-4476" />
+        <rdf:li rdf:resource="http://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2015-4500" />
+        <rdf:li rdf:resource="http://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2015-4501" />
+     </rdf:Seq>
+    </items>
+    <dc:date>2015-09-27T04:50:00Z</dc:date>
+    <dc:language>en-us</dc:language>
+    <dc:rights>This material is not copywritten and may be freely used, however, attribution is requested.</dc:rights>
+  </channel>
+  <item rdf:about="http://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2015-4476">
+    <title>CVE-2015-4476 (firefox)</title>
+    <link>http://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2015-4476</link>
+    <description>Mozilla Firefox before 41.0 on Android allows user-assisted remote attackers to spoof address-bar attributes by leveraging lack of navigation after a paste of a URL with a nonstandard scheme, as demonstrated by spoofing an SSL attribute.</description>
+    <dc:date>2015-09-24T04:59:00Z</dc:date>
+  </item>
+  <item rdf:about="http://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2015-4500">
+    <title>CVE-2015-4500 (firefox, firefox_esr)</title>
+    <link>http://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2015-4500</link>
+    <description>Multiple unspecified vulnerabilities in the browser engine in Mozilla Firefox before 41.0 and Firefox ESR 38.x before 38.3 allow remote attackers to cause a denial of service (memory corruption and application crash) or possibly execute arbitrary code via unknown vectors.</description>
+    <dc:date>2015-09-24T04:59:02Z</dc:date>
+  </item>
+  <item rdf:about="http://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2015-4501">
+    <title>CVE-2015-4501 (firefox)</title>
+    <link>http://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2015-4501</link>
+    <description>Multiple unspecified vulnerabilities in the browser engine in Mozilla Firefox before 41.0 allow remote attackers to cause a denial of service (memory corruption and application crash) or possibly execute arbitrary code via unknown vectors.</description>
+    <dc:date>2015-09-24T04:59:03Z</dc:date>
+  </item>
+</rdf:RDF>
+    eos
+  end
+end
+
+class VulnerabilityMock
+  def self.generate
+    mocker = VulnerabilityMock.new
+    id = mocker.identifier
+
+    CVE::Vulnerability.new({
+      :identifier => id,
+      :title => mocker.title(id),
+      :link => mocker.link(id),
+      :description => mocker.description,
+      :date => mocker.date
+    })
+  end
+
+  def identifier
+    year = Time.now.year
+    "CVE-#{rand(2000..year)}-#{rand(0..9999)}"
+  end
+
+  def title(id=nil)
+    (id || identifier) + ' (' + ['Lorem Ipsum', 'Some vulnerability', 'Your favourite software',
+      'Firefox', 'Chromium', 'Thunderbird'].sample.split('').shuffle.join + ')'
+  end
+
+  def link(id=nil)
+    'http://web.nvd.nist.gov/view/vuln/detail?vulnId=' + (id || identifier)
+  end
+
+  def description
+    ['Hakuna Matata', 'What a wonderful phrase', 'Ain\'t no passing phrase', 'It means no worries',
+      'for the rest of your days', 'It\'s our problem-free philosophy', 'Yeah. That\'s our motto',
+      'What\'s a motto?'].sample.split('').shuffle.join
+  end
+
+  def date
+    Time.now
+  end
+end

data/spec/vulnerability_spec.rb

@@ -0,0 +1,137 @@
+require_relative 'spec_helper'
+
+describe CVE::Vulnerability do
+  mocker = VulnerabilityMock.new
+  cve_vul_obj = VulnerabilityMock.generate
+
+  it 'should not error when passing a valid hash' do
+    id = mocker.identifier
+
+    item = CVE::Vulnerability.new({
+      :identifier => id,
+      :title => mocker.title(id),
+      :link => mocker.link(id),
+      :description => mocker.description,
+      :date => mocker.date
+    })
+
+    expect(item).to be_a(CVE::Vulnerability)
+  end
+
+  it 'should compare to a CVE with the same identifier and link' do
+    id = mocker.identifier
+    link = mocker.link(id)
+    item = CVE::Vulnerability.new({
+      :identifier => id,
+      :title => mocker.title(id),
+      :link => link,
+      :description => mocker.description,
+      :date => mocker.date
+    })
+
+    item_roughly_identical = CVE::Vulnerability.new({
+        :identifier => id,
+        :title => mocker.title(id),
+        :link => link,
+        :description => mocker.description,
+        :date => mocker.date
+    })
+
+    expect(item.equal?(item_roughly_identical, false)).to equal(true)
+  end
+
+  it 'should compare strictly to an identical CVE' do
+    id = mocker.identifier
+    item = CVE::Vulnerability.new({
+      :identifier => id,
+      :title => mocker.title(id),
+      :link => mocker.link(id),
+      :description => mocker.description,
+      :date => mocker.date
+    })
+
+    item_identical = item.clone
+
+    expect(item.equal?(item_identical, true)).to equal(true)
+  end
+
+  it 'should fail compare strictly to an unidentical CVE with the same ID' do
+    id = mocker.identifier
+    item = CVE::Vulnerability.new({
+      :identifier => id,
+      :title => mocker.title(id),
+      :link => mocker.link(id),
+      :description => mocker.description,
+      :date => mocker.date
+    })
+
+    item_unidentical = CVE::Vulnerability.new({
+      :identifier => id,
+      :title => mocker.title(id),
+      :link => mocker.link(id),
+      :description => mocker.description,
+      :date => mocker.date
+    })
+
+    expect(item.equal?(item_unidentical, true)).to equal(false)
+  end
+
+  it 'should fail compare to an unidentical CVE' do
+    id = mocker.identifier
+    item = CVE::Vulnerability.new({
+      :identifier => id,
+      :title => mocker.title(id),
+      :link => mocker.link(id),
+      :description => mocker.description,
+      :date => mocker.date
+    })
+
+    id = mocker.identifier
+    item_unidentical = CVE::Vulnerability.new({
+      :identifier => id,
+      :title => mocker.title(id),
+      :link => mocker.link(id),
+      :description => mocker.description,
+      :date => mocker.date
+    })
+
+    expect(item.equal?(item_unidentical, false)).to equal(false)
+  end
+
+  it 'should be able to extract a single software from the title' do
+    title = 'CVE-2015-123 (firefox)'
+    extract = cve_vul_obj.extract_software_from_title(title)
+
+    expect(extract.count).to eq(1)
+    expect(extract[0]).to eq('firefox')
+  end
+
+  it 'should be able to extract many software from the title' do
+    title = 'CVE-2015-123 (firefox, firefox_esr, flash)'
+    extract = cve_vul_obj.extract_software_from_title(title)
+
+    expect(extract.count).to eq(3)
+    expect(extract[0]).to eq('firefox')
+    expect(extract[1]).to eq('firefox_esr')
+    expect(extract[2]).to eq('flash')
+  end
+
+  it 'should return nil for affected software when not included' do
+    title = 'CVE-2015-123'
+    extract = cve_vul_obj.extract_software_from_title(title)
+
+    expect(extract).to eq(nil)
+  end
+
+  it 'should error when not passing a hash' do
+    expect{ CVE::Vulnerability.new('') }.to raise_error(RuntimeError)
+  end
+
+  it 'should error when passing an empty hash' do
+    expect{ CVE::Vulnerability.new({}) }.to raise_error(RuntimeError)
+  end
+
+  it 'should error when passing a hash with missing keys' do
+    expect{ CVE::Vulnerability.new({:identifier => 'abc', :title => 'title'}) }.to raise_error(RuntimeError)
+  end
+end

metadata

@@ -0,0 +1,54 @@
+--- !ruby/object:Gem::Specification
+name: cve_crawler
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Jos Ahrens
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-09-27 00:00:00.000000000 Z
+dependencies: []
+description: A periodic crawler that fetches the latest CVE additions, parses them,
+  and filters them
+email: gems@zarth.us
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/cve_crawler/cve_core.rb
+- lib/cve_crawler/cve_crawler.rb
+- lib/cve_crawler/cve_filter.rb
+- lib/cve_crawler/cve_parser.rb
+- lib/cve_crawler/cve_vulnerability.rb
+- spec/crawler_spec.rb
+- spec/filter_spec.rb
+- spec/parser_spec.rb
+- spec/spec_helper.rb
+- spec/vulnerability_spec.rb
+homepage: https://github.com/zarthus/ruby-cve-crawler
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: CVE Crawler
+test_files: []