threatinator 0.1.6 → 0.2.0

data/spec/support/shared/feeds.rb

@@ -16,12 +16,21 @@ end
 
 shared_context 'a parsed feed' do 
   # expects :observer
-  let(:events) { observer.map { |status, record, events| events } }
-  let(:records) { observer.map { |status, record, events| record } }
+  let(:events) { observer.events }
+  let(:records) { observer.records }
   let(:num_records) { observer.count }
   let(:num_records_filtered) { observer.num_records_filtered }
   let(:num_records_parsed) { observer.num_records_parsed }
   let(:num_records_missed) { observer.num_records_missed }
+  let(:num_records_errored) { observer.num_records_errored }
+
+  it "should have 0 error records" do
+    expect(num_records_errored).to eq(0)
+  end
+
+  it "should have missed 0 records" do
+    expect(num_records_missed).to eq(0)
+  end
 end
 
 shared_context 'for feeds', :feed => lambda { true } do
@@ -114,7 +123,7 @@ module FeedHelpers
   class FeedRunnerObserver
     include Enumerable
     attr_reader :records, :statuses, :events, :num_records_filtered, 
-      :num_records_missed, :num_records_parsed
+      :num_records_missed, :num_records_parsed, :num_records_errored
 
     def initialize
       @records = []
@@ -123,6 +132,7 @@ module FeedHelpers
       @num_records_filtered = 0
       @num_records_parsed = 0
       @num_records_missed = 0
+      @num_records_errored = 0
     end
 
     def each
@@ -149,6 +159,11 @@ module FeedHelpers
         @statuses << :parsed
         @events << args.shift
         @num_records_parsed += 1
+      when :record_error
+        @records << args.shift
+        @statuses << :error
+        @events << []
+        @num_records_errored += 1
       end
     end
   end
@@ -174,7 +189,7 @@ module FeedHelpers
         before :each do
           sio = StringIO.new(data)
           feed_runner.add_observer(observer)
-          feed_runner.run(:io => sio)
+          feed_runner.run(:io => sio, :skip_decoding => true)
           @status, @record, @events = observer.first
         end
 

data/spec/support/shared/model/collection.rb

@@ -0,0 +1,164 @@
+require 'spec_helper'
+
+shared_examples_for "a model collection" do
+  # expect generate_ten_valid_members, generate_invalid_members
+  let(:collection) { described_class.new }
+
+  describe "initialize" do
+    context "when no parameters are provided" do
+      specify "the collection is empty" do
+        x = described_class.new
+        expect(x).to be_empty
+      end
+    end
+
+    context "when an array of members is provided" do
+      it "calls #<< for each member, adding it to the collection" do
+        members = generate_ten_valid_members()
+        expect_any_instance_of(described_class).to receive(:<<).exactly(10).times.and_call_original
+        x = described_class.new(members)
+        expect(x.to_a).to match_array(members)
+      end
+    end
+  end
+  describe "#<<(member)" do
+    it "adds the member to the collection" do
+      member = generate_ten_valid_members()[0]
+      expect(collection.include?(member)).to eq(false)
+      collection << member
+      expect(collection.include?(member)).to eq(true)
+    end
+
+    it "does not add the member if it is equal to any member of the collection" do
+      member = generate_ten_valid_members()[0]
+      expect(collection.count).to eq(0)
+      collection << member
+      expect(collection.count).to eq(1)
+      collection << member
+      expect(collection.count).to eq(1)
+    end
+
+    it "raises InvalidAttributeError if the added member is not valid for the collection" do
+      expect {
+        collection << generate_invalid_members()[0]
+      }.to raise_error(Threatinator::Exceptions::InvalidAttributeError)
+    end
+  end
+  
+  describe "#valid_member?(member)" do
+    it "returns true if the member is valid" do
+      expect(collection.valid_member?(generate_ten_valid_members()[0])).to eq(true)
+    end
+    it "returns false if the member is not valid" do
+      expect(collection.valid_member?(generate_invalid_members()[0])).to eq(false)
+    end
+  end
+
+  describe "#include?(member)" do
+    let(:member) { generate_ten_valid_members()[0] }
+    it "returns true if the member is in the collection" do
+      collection << member
+      expect(collection).to include(member)
+    end
+
+    it "returns false if the member is not in the collection" do
+      expect(collection).not_to include(member)
+    end
+  end
+
+  describe "#empty?" do
+    it "returns true when the collection is empty" do
+      expect(collection).to be_empty
+    end
+    it "returns false when something has been added to the collection" do
+      member = generate_ten_valid_members()[0]
+      collection << member
+      expect(collection).not_to be_empty
+    end
+  end
+
+  describe "#count" do
+    it "returns 0 when the collection is empty" do
+      expect(collection.count).to eq(0)
+    end
+
+    it "returns the number of members in the collection" do
+      members = generate_ten_valid_members()
+      members.each do |member|
+        collection << member
+      end
+      expect(collection.count).to eq(members.count)
+    end
+  end
+
+  [:to_ary, :to_a].each do |method_name|
+    describe "##{method_name}" do 
+      it "returns an empty array instance when the collection is empty" do
+        expect(collection.send(method_name)).to eq([])
+      end
+
+      it "returns an array containing members that have been added" do
+        members = generate_ten_valid_members()
+        members.each do |member|
+          collection << member
+        end
+        expect(collection.send(method_name)).to match_array(members)
+      end
+    end
+  end
+
+  describe "#each" do
+    it "returns an Enumerator if no block is provided" do
+      expect(collection.each).to be_a(::Enumerator)
+    end
+
+    it "yields each member of the collection" do
+      members = generate_ten_valid_members()
+      members.each do |member|
+        collection << member
+      end
+
+      actual = []
+      collection.each do |member|
+        actual << member
+      end
+
+      expect(actual).to match_array(members)
+    end
+  end
+
+  describe "#==(other)" do
+    before :each do
+    end
+
+    it "returns true when compared to itself" do
+      collection = described_class.new(generate_ten_valid_members())
+      expect(collection).to be == collection
+    end
+
+    it "returns true when compared to a collection with equal members" do
+      collection = described_class.new(generate_ten_valid_members())
+      collection2 = described_class.new(generate_ten_valid_members())
+      expect(collection).to be == collection2
+    end
+
+    it "returns false if the number of members is different" do
+      collection = described_class.new(generate_ten_valid_members()[0..5])
+      collection2 = described_class.new(generate_ten_valid_members())
+      expect(collection).not_to be == collection2
+    end
+
+    it "returns false if the members are different" do
+      collection = described_class.new(generate_ten_valid_members()[0..4])
+      collection2 = described_class.new(generate_ten_valid_members()[5..9])
+      expect(collection).not_to be == collection2
+    end
+
+    it "returns false when compared to something other than a collection" do
+      members = generate_ten_valid_members()
+      collection = described_class.new(members)
+      expect(collection).not_to be == members
+    end
+  end
+end
+

data/spec/threatinator/actions/run/action_spec.rb

@@ -31,35 +31,52 @@ describe Threatinator::Actions::Run::Action do
     end
 
     describe "#exec" do
-      before :each do
-        @ret = action.exec
-      end
 
       it "queries the feed registry for the provider and name" do
+        action.exec
         expect(feed_registry).to have_received(:get).with("my_provider", "my_name")
       end
 
       it "builds the output using config.output.build_output" do
+        action.exec
         expect(config.output).to have_received(:build_output)
       end
 
       it "runs the feed" do
+        action.exec
         expect(feed_runner).to have_received(:run)
       end
-    end
 
-    context "when no observer is configured" do
-      before :each do
-        config.observers = [ ]
+      context "when a record was missed" do
+        let(:status_observer) { Threatinator::Actions::Run::StatusObserver.new }
+        before :each do
+          allow(Threatinator::Actions::Run::StatusObserver).to receive(:new).and_return(status_observer)
+          allow(status_observer).to receive(:missed?).and_return(true)
+        end
+
+        it "logs an error message" do
+          expect(action.logger).to receive(:error).with(/records were MISSED/)
+          action.exec
+        end
       end
-      it "does not add any observers to the feed runner" do
-        expect(feed_runner).not_to receive(:add_observer)
-        action.exec
+
+      context "when a record had an error" do
+        let(:status_observer) { Threatinator::Actions::Run::StatusObserver.new }
+        before :each do
+          allow(Threatinator::Actions::Run::StatusObserver).to receive(:new).and_return(status_observer)
+          allow(status_observer).to receive(:errors?).and_return(true)
+        end
+
+        it "logs an error message" do
+          expect(action.logger).to receive(:error).with(/records had errors/)
+          action.exec
+        end
       end
     end
 
     context "when configured with an observer" do
       before :each do
+        allow(feed_runner).to receive(:add_observer)
         config.observers = [ observer ]
       end
 

data/spec/threatinator/actions/run/coverage_observer_spec.rb

@@ -14,6 +14,8 @@ describe Threatinator::Actions::Run::CoverageObserver do
   let(:filename) { File.join(@tmpdir, "coverage.csv") }
   let(:observer) { described_class.new(filename) }
 
+  it_should_behave_like "a FeedRunner observer"
+
   context "#update(:start)" do
     it "creates the file specified by filename" do
       expect(File.exist?(filename)).to eq(false)
@@ -35,7 +37,7 @@ describe Threatinator::Actions::Run::CoverageObserver do
 
       specify "the first line is the header" do
         data = File.read(filename)
-        expect(data.lines.to_a.first).to eq("status,event_count,line_number,pos_start,pos_end,data\n")
+        expect(data.lines.to_a.first).to eq("status,event_count,line_number,pos_start,pos_end,data,message\n")
       end
 
       it "closes the file so that no more records will be written" do
@@ -65,7 +67,8 @@ describe Threatinator::Actions::Run::CoverageObserver do
         line_number: "23",
         pos_start: "99",
         pos_end: "105",
-        data: '"foobar\r\n"'
+        data: '"foobar\r\n"',
+        message: ''
       )
     end
   end
@@ -86,7 +89,8 @@ describe Threatinator::Actions::Run::CoverageObserver do
         line_number: "22",
         pos_start: "98",
         pos_end: "104",
-        data: '"blabla\r\n"'
+        data: '"blabla\r\n"',
+        message: ''
       )
     end
   end
@@ -109,7 +113,38 @@ describe Threatinator::Actions::Run::CoverageObserver do
         line_number: "1",
         pos_start: "0",
         pos_end: "10",
-        data: '"woofwoof\r\n"'
+        data: '"woofwoof\r\n"',
+        message: ''
+      )
+    end
+  end
+
+  context "#update(:record_error, record, errors)" do
+    before :each do
+      observer.update(:start)
+    end
+
+    let(:record) { build(:record, line_number: 1, pos_start: 0, pos_end: 10, data: "woofwoof\r\n") }
+    let(:errors) { 
+      [
+        Threatinator::Exceptions::EventBuildError.new("error 1"),
+        Threatinator::Exceptions::EventBuildError.new("error 2"),
+        Threatinator::Exceptions::EventBuildError.new("error 3")
+      ]
+    }
+
+    it "writes a csv entry to the file indicating it encountered an error, with the error messages" do
+      observer.update(:record_error, record, errors)
+      observer.update(:end)
+      csv = CSV.read(filename, headers: true, header_converters: :symbol)
+      expect(csv[-1].to_hash).to eq(
+        status: "error",
+        event_count: "0",
+        line_number: "1",
+        pos_start: "0",
+        pos_end: "10",
+        data: '"woofwoof\r\n"',
+        message: 'error 1, error 2, error 3'
       )
     end
   end

data/spec/threatinator/actions/run/status_observer_spec.rb

@@ -0,0 +1,86 @@
+require 'spec_helper'
+require 'threatinator/actions/run/status_observer'
+
+describe Threatinator::Actions::Run::StatusObserver do
+  let(:observer) { described_class.new() }
+  let(:record) { build(:record, line_number: 23, pos_start: 99, pos_end: 105, data: "foobar\r\n") }
+
+  it_should_behave_like "a FeedRunner observer"
+
+  describe "#filtered" do
+    it "returns the number of records that have been filtered" do
+      expect(observer.filtered).to eq(0)
+      observer.update(:record_filtered, record)
+      expect(observer.filtered).to eq(1)
+    end
+  end
+
+  describe "#filtered?" do
+    it "returns true if any records were filtered, false otherwise" do
+      expect(observer.filtered?).to eq(false)
+      observer.update(:record_filtered, record)
+      expect(observer.filtered?).to eq(true)
+    end
+  end
+
+  describe "#missed" do
+    it "returns the number of records that have been missed" do
+      expect(observer.missed).to eq(0)
+      observer.update(:record_missed, record)
+      expect(observer.missed).to eq(1)
+    end
+  end
+
+  describe "#missed?" do
+    it "returns true if any records were missed, false otherwise" do
+      expect(observer.missed?).to eq(false)
+      observer.update(:record_missed, record)
+      expect(observer.missed?).to eq(true)
+    end
+  end
+
+  describe "#parsed" do
+    it "returns the number of records that have been parsed" do
+      expect(observer.parsed).to eq(0)
+      observer.update(:record_parsed, record)
+      expect(observer.parsed).to eq(1)
+    end
+  end
+
+  describe "#parsed?" do
+    it "returns true if any records were parsed, false otherwise" do
+      expect(observer.parsed?).to eq(false)
+      observer.update(:record_parsed, record)
+      expect(observer.parsed?).to eq(true)
+    end
+  end
+
+  describe "#errors" do
+    it "returns the number of records that had errors" do
+      expect(observer.errors).to eq(0)
+      observer.update(:record_error, record)
+      expect(observer.errors).to eq(1)
+    end
+  end
+
+  describe "#errors?" do
+    it "returns true if any records had errors, false otherwise" do
+      expect(observer.errors?).to eq(false)
+      observer.update(:record_error, record)
+      expect(observer.errors?).to eq(true)
+    end
+  end
+
+
+  describe "#total" do
+    it "returns the total number of records that were parsed, missed, filtered, and errored" do
+      expect(observer.total).to eq(0)
+      10.times { observer.update(:record_parsed, record) }
+      10.times { observer.update(:record_missed, record) }
+      10.times { observer.update(:record_filtered, record) }
+      10.times { observer.update(:record_error, record) }
+      expect(observer.total).to eq(40)
+    end
+  end
+end
+