cve_crawler 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d08b715392de5eb5c6a654cdfaf1ad9402cf3f3b
-  data.tar.gz: ec204ca9f33f79223f380c701b41c869f1754113
+  metadata.gz: 31c7faeb4c2a75bb522bd206fc7cde12de39fee6
+  data.tar.gz: b0f13771e93ec38491348ef6eb0b35f537c0a843
 SHA512:
-  metadata.gz: 3cf8e02f4afff547d1331191df92d88dd3aeb91ae577e650bc8aeb41ad1e5fcbc23c862e834c8661d3a94f7ed4d01e28a507d1578de1fe134de84c76186716ba
-  data.tar.gz: 1c43b325124a7006e2eb5760f44f29c88a6ede065a2be765d2ab3346d60e68b22ff60a12619a3dd3635a8a2ce0d1974a6a0a82cd3b355c1adc595c3bac6b41ec
+  metadata.gz: 3022536ed2bf42f437d8942a883abaa9e3125fe6ce4ab85056f776488da02080f3a370d742180a5faa88f4084e3e44023d93789bd32f00decd2834742f554085
+  data.tar.gz: c701cf723c1eed68b70261c92f08400a5a84495445bb9936dee050a3a821f1349e744b2c28c36beebb6ee1c0bf809c90d80f02ddf503a0bd3fd0857c9006fa9d

data/lib/cve_crawler/cve_core.rb

@@ -4,7 +4,7 @@ require 'cve_crawler/cve_parser'
 
 module CVE
   VERSION_MAJOR = 0
-  VERSION_MINOR = 2
+  VERSION_MINOR = 3
   VERSION_BUILD = 0
   VERSION = "#{VERSION_MAJOR}.#{VERSION_MINOR}.#{VERSION_BUILD}".freeze
 
@@ -18,6 +18,8 @@ module CVE
       @parser = Parser.new(filters)
     end
 
+    attr_reader :crawler, :parser
+
     def fetch
       body = crawl.body
       parse(body)

data/lib/cve_crawler/cve_filter.rb

@@ -5,7 +5,7 @@ module CVE
       @history_size = history_size
     end
 
-    attr_reader :history_size
+    attr_accessor :history, :history_size
 
     def filter(contents)
       return true unless contents.is_a?(Vulnerability)
@@ -23,8 +23,8 @@ module CVE
     end
 
     def history_check_limit
-      if @history.length >= @history_size
-        @history.drop(1)
+      if @history.length > @history_size
+        @history = @history.drop(@history.length - @history_size)
       end
     end
 

data/lib/cve_crawler/cve_parser.rb

@@ -6,7 +6,6 @@ module CVE
   class Parser
     def initialize(filters=nil)
       @filters = []
-
       @filters << CVE::Filter.new
 
       if filters.is_a?(Array)
@@ -18,6 +17,8 @@ module CVE
       end
     end
 
+    attr_reader :filters
+
     def parse(content)
       results = parse_rss_feed(content)
 

data/lib/cve_crawler/cve_vulnerability.rb

@@ -55,6 +55,17 @@ module CVE
       "#{@title} - #{@link}"
     end
 
+    def to_hash
+      {
+        :identifier => @identifier,
+        :title => @title,
+        :link => @link,
+        :description => @description,
+        :date => @date,
+        :affected_software => @affected_software
+      }
+    end
+
     def inspect
       "#<CVE::Vulnerability id=#{@identifier} affected=#{affected_count}>"
     end

data/spec/core_spec.rb

@@ -0,0 +1,61 @@
+require_relative 'spec_helper'
+
+# Not exactly what it says on the tin: Everything here is faked to simulate an environment
+# where you can dynamically crawl, parse, filter and generate new vulnerabilities like the
+# real RSS feed would.  This simply simulates that behaviour to verify the system as a whole
+# still functions.  These tests failing might not necessarily represent a failure in the core
+# but are more likely to represent a failure in one of the components.
+describe CVE::Core do
+  core = CoreMock.new
+
+  it 'should fetch new results without error' do
+    expect{ core.fetch }.not_to raise_error
+  end
+
+  it 'should list new vulnerabilities when CVE::Core#fetch is called and new vulnerabilities exist' do
+    result = core.fetch
+
+    expect(result.count).not_to eq(0)
+  end
+
+  it 'should return a list of CVE::Vulnerability when vulnerabilities have been found' do
+    result = core.fetch
+
+    result.each do |res|
+      expect(res).to be_a(CVE::Vulnerability)
+    end
+  end
+
+  it 'should not list old vulnerabilities when CVE::Core#fetch is called' do
+    tmp_core = create_core_object
+
+    old_cves = tmp_core.fetch
+    new_cves = tmp_core.fetch
+
+    new_cves.each do |new|
+      old_cves.each do |old|
+        expect(new.identifier).not_to eq(old.identifier)
+        expect(new.equal?(old)).to eq(false)
+      end
+    end
+  end
+
+  it 'should append as many items to the history as were fetched' do
+    tmp_core = create_core_object
+
+    result = tmp_core.fetch
+
+    expect(tmp_core.parser.filters[0].history.count).to eq(result.count)
+  end
+
+  it 'should append new items to the history of the filter' do
+    tmp_core = create_core_object
+
+    tmp_core.fetch
+    current_history_size = tmp_core.parser.filters[0].history.count
+    result = tmp_core.fetch
+
+    expect(tmp_core.parser.filters[0].history.count).not_to eq(current_history_size)
+    expect(tmp_core.parser.filters[0].history.count).to eq(current_history_size + result.count)
+  end
+end

data/spec/filter_spec.rb

@@ -16,4 +16,69 @@ describe CVE::Filter do
       expect(filter.filter(VulnerabilityMock.generate)).to equal(false)
     end
   end
+
+  it 'should not exceed the history limit imposed' do
+    filter = CVE::Filter.new
+
+    filter_fill_history(filter, filter.history_size + 1)
+
+    expect(filter.history.count).to eq(filter.history_size)
+  end
+
+  it 'should not exceed the history limit customly set' do
+    filter = CVE::Filter.new(3)
+    expect(filter.history_size).to eq(3)
+
+    filter_fill_history(filter, filter.history_size + 1)
+
+    expect(filter.history.count).to eq(filter.history_size)
+  end
+
+  it 'should not store duplicate history entries' do
+    filter = CVE::Filter.new
+    cve = VulnerabilityMock.generate
+
+    filter.filter(cve)
+    filter.filter(cve)
+
+    expect(filter.history.count).to eq(1)
+  end
+
+  it 'should remove older history items first' do
+    filter = CVE::Filter.new(1)
+    cve = VulnerabilityMock.generate
+
+    filter.filter(cve)
+    filter.filter(VulnerabilityMock.generate)
+
+    expect(filter.history[0]).not_to eq(cve)
+  end
+
+  it 'should remove older history items first' do
+    filter = CVE::Filter.new(5)
+    batches = []
+    batch = []
+
+    2.times do
+      5.times do
+        batch << VulnerabilityMock.generate
+      end
+      batches << batch
+      batch = []
+    end
+
+    expect {
+      batches.each do |vulns|
+        vulns.each do |vuln|
+          raise 'Unique filtered content was determined to be filtered?' if filter.filter(vuln) == true
+        end
+      end
+    }.not_to raise_error
+
+    history_ok = true
+    5.times do |i|
+      history_ok = history_ok && filter.history[i] == batches[1][i].identifier
+    end
+    expect(history_ok).to equal(true)
+  end
 end

data/spec/parser_spec.rb

@@ -22,6 +22,21 @@ describe CVE::Parser do
     expect{ parser.parse('Not xml') }.to raise_error(RuntimeError)
   end
 
+  it 'should parse individual items' do
+    vulns = []
+
+    3.times do
+      vulns << VulnerabilityMock.generate
+    end
+
+    contents = RSSMock.new(vulns)
+    items = parser.parse_items(contents)
+
+    3.times do |i|
+      expect(items[i].equal?(vulns[i])).to eq(true)
+    end
+  end
+
   it 'should extract the CVE identifier from a title' do
     title = VulnerabilityMock.new.title
 

data/spec/spec_helper.rb

@@ -1,84 +1,21 @@
 require 'rspec'
 require_relative File.join('..', 'lib', 'cve_crawler')
+require_relative 'mocks/rss_mock'
+require_relative 'mocks/cve_crawler_mock'
+require_relative 'mocks/vulnerability_mock'
 
-class CrawlerResultMock
-  def xml_full
-    <<-eos
-<?xml version="1.0" encoding="UTF-8"?>
-<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns="http://purl.org/rss/1.0/" xmlns:dc="http://purl.org/dc/elements/1.1/">
-  <channel rdf:about="http://web.nvd.nist.gov/view/vuln/search">
-    <title>National Vulnerability Database</title>
-    <link>http://web.nvd.nist.gov/view/vuln/search</link>
-    <description>This feed contains the most recent CVE cyber vulnerabilities published within the National Vulnerability Database.</description>
-    <items>
-      <rdf:Seq>
-        <rdf:li rdf:resource="http://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2015-4476" />
-        <rdf:li rdf:resource="http://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2015-4500" />
-        <rdf:li rdf:resource="http://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2015-4501" />
-     </rdf:Seq>
-    </items>
-    <dc:date>2015-09-27T04:50:00Z</dc:date>
-    <dc:language>en-us</dc:language>
-    <dc:rights>This material is not copywritten and may be freely used, however, attribution is requested.</dc:rights>
-  </channel>
-  <item rdf:about="http://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2015-4476">
-    <title>CVE-2015-4476 (firefox)</title>
-    <link>http://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2015-4476</link>
-    <description>Mozilla Firefox before 41.0 on Android allows user-assisted remote attackers to spoof address-bar attributes by leveraging lack of navigation after a paste of a URL with a nonstandard scheme, as demonstrated by spoofing an SSL attribute.</description>
-    <dc:date>2015-09-24T04:59:00Z</dc:date>
-  </item>
-  <item rdf:about="http://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2015-4500">
-    <title>CVE-2015-4500 (firefox, firefox_esr)</title>
-    <link>http://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2015-4500</link>
-    <description>Multiple unspecified vulnerabilities in the browser engine in Mozilla Firefox before 41.0 and Firefox ESR 38.x before 38.3 allow remote attackers to cause a denial of service (memory corruption and application crash) or possibly execute arbitrary code via unknown vectors.</description>
-    <dc:date>2015-09-24T04:59:02Z</dc:date>
-  </item>
-  <item rdf:about="http://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2015-4501">
-    <title>CVE-2015-4501 (firefox)</title>
-    <link>http://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2015-4501</link>
-    <description>Multiple unspecified vulnerabilities in the browser engine in Mozilla Firefox before 41.0 allow remote attackers to cause a denial of service (memory corruption and application crash) or possibly execute arbitrary code via unknown vectors.</description>
-    <dc:date>2015-09-24T04:59:03Z</dc:date>
-  </item>
-</rdf:RDF>
-    eos
-  end
-end
-
-class VulnerabilityMock
-  def self.generate
-    mocker = VulnerabilityMock.new
-    id = mocker.identifier
-
-    CVE::Vulnerability.new({
-      :identifier => id,
-      :title => mocker.title(id),
-      :link => mocker.link(id),
-      :description => mocker.description,
-      :date => mocker.date
-    })
-  end
-
-  def identifier
-    year = Time.now.year
-    "CVE-#{rand(2000..year)}-#{rand(0..9999)}"
+def filter_fill_history(filter, size)
+  size.times do
+    filter.filter(VulnerabilityMock.generate)
   end
 
-  def title(id=nil)
-    (id || identifier) + ' (' + ['Lorem Ipsum', 'Some vulnerability', 'Your favourite software',
-      'Firefox', 'Chromium', 'Thunderbird'].sample.split('').shuffle.join + ')'
-  end
-
-  def link(id=nil)
-    'http://web.nvd.nist.gov/view/vuln/detail?vulnId=' + (id || identifier)
-  end
+  filter
+end
 
-  def description
-    ['Hakuna Matata', 'What a wonderful phrase', 'Ain\'t no passing phrase', 'It means no worries',
-      'for the rest of your days', 'It\'s our problem-free philosophy', 'Yeah. That\'s our motto',
-      'What\'s a motto?'].sample.split('').shuffle.join
-  end
+def create_core_object
+  # Ensure a clear history and a large history buffer to avoid tests failing when the limit is reached
+  tmp_core = CoreMock.new
+  tmp_core.parser.filters[0].history_size = 50 if tmp_core.parser.filters[0].history_size < 50
 
-  def date
-    Time.now
-  end
+  tmp_core
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cve_crawler
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Jos Ahrens
@@ -22,6 +22,7 @@ files:
 - lib/cve_crawler/cve_filter.rb
 - lib/cve_crawler/cve_parser.rb
 - lib/cve_crawler/cve_vulnerability.rb
+- spec/core_spec.rb
 - spec/crawler_spec.rb
 - spec/filter_spec.rb
 - spec/parser_spec.rb