threatinator 0.1.6 → 0.2.0

data/spec/threatinator/feed_builder_spec.rb

@@ -466,7 +466,6 @@ parse_eachline(:separator => "\n") {}'
           feed_loader.call(feed_string).build
         end.to raise_error { |e| 
           expect(e).to be_a(Threatinator::Exceptions::InvalidAttributeError)
-          expect(e.attribute).to eq(:provider)
         }
       end
 
@@ -479,7 +478,6 @@ parse_eachline(:separator => "\n") {}'
           feed_loader.call(feed_string).build
         end.to raise_error { |e| 
           expect(e).to be_a(Threatinator::Exceptions::InvalidAttributeError)
-          expect(e.attribute).to eq(:name)
         }
       end
 
@@ -492,7 +490,6 @@ parse_eachline(:separator => "\n") {}'
           feed_loader.call(feed_string).build
         end.to raise_error { |e| 
           expect(e).to be_a(Threatinator::Exceptions::InvalidAttributeError)
-          expect(e.attribute).to eq(:fetcher_builder)
         }
       end
 

data/spec/threatinator/feed_runner_spec.rb

@@ -2,21 +2,79 @@ require 'spec_helper'
 require 'threatinator/feed_runner'
 
 describe Threatinator::FeedRunner do
-  let(:output_formatter ) { double("formatter") }
-  let(:fetcher) { double("fetcher") }
-  let(:fetcher_builder) { lambda { fetcher } }
+  class DummyParser < Threatinator::Parser
+    def initialize(records, opts = {})
+      @records = records
+    end
+
+    def run(io)
+      @records.each do |record|
+        yield(record)
+      end
+    end
+  end
+
+  class DummyFetcher < Threatinator::Fetcher
+    def initialize(io, opts = {})
+      @io = io
+    end
+
+    def fetch
+      return @io
+    end
+  end
+
+  class DummyDecoder < Threatinator::Decoder
+    def initialize(io)
+      @io = io
+    end
+
+    def decode(arg_io)
+      return @io
+    end
+  end
+
+  class TestObserver
+    attr_reader :updates
+    def initialize
+      @updates = []
+    end
+
+    def update(*args)
+      @updates << args
+    end
+  end
+
+  class DummyOutput < Threatinator::Output
+    def handle_event(event); end
+    def finish; end
+  end
+
+  def create_mock_io(name = "io")
+    ret = double(name) 
+    allow(ret).to receive(:close)
+    allow(ret).to receive(:closed?).and_return(false)
+    ret
+  end
 
-  let(:io) { double("io") }
-  let(:parser) { double("parser") }
-  let(:parser_builder) { lambda { parser} }
+  let(:output_formatter ) { DummyOutput.new(nil) }
+  let(:io) { create_mock_io("io") }
+  let(:fetcher) { DummyFetcher.new(io) }
 
-  let(:filter_builders) { [] }
-  let(:decoder_builders) { [] }
+  let(:record1) { Threatinator::Record.new('a1') }
+  let(:record2) { Threatinator::Record.new('a2') }
+  let(:record3) { Threatinator::Record.new('a3') }
+  let(:records) { [ record1, record2, record3 ] }
+  let(:parser) { DummyParser.new(records) }
+  let(:parser_block) { lambda { |*args| } }
+
+  let(:filters) { [] }
+  let(:decoders) { [] }
 
   let(:feed) {
-    build(:feed, fetcher_builder: fetcher_builder, 
-          parser_builder: parser_builder, filter_builders: filter_builders,
-          decoder_builders: decoder_builders
+    build(:feed, fetcher: fetcher, 
+          parser: parser, filters: filters,
+          decoders: decoders, parser_block: parser_block
          )
   }
 
@@ -61,93 +119,219 @@ describe Threatinator::FeedRunner do
 
   context "an instance" do
     let(:feed_runner) { described_class.new(feed, output_formatter) }
+    let(:observer) { TestObserver.new }
+
+    before :each do
+      feed_runner.add_observer(observer)
+      allow(observer).to receive(:update).and_call_original
+    end
 
     describe "#run" do
-      context "fetching data" do
-        before :each do
-          allow(parser).to receive(:run).with(io)
+      it "notifies the observer with :start before anything else" do
+        expect(observer.updates.first).to be_nil
+        feed_runner.run()
+        expect(observer.updates.first).to eq([:start])
+      end
+
+      it "closes the last IO instance" do
+        allow(fetcher).to receive(:fetch).and_return(io)
+        expect(io).to receive(:close)
+        feed_runner.run()
+      end
+
+      it "fetches, decodes, and then parses records" do
+        expect(observer).to receive(:update).with(:start).ordered
+        expect(observer).to receive(:update).with(:start_fetch).ordered
+        expect(observer).to receive(:update).with(:end_fetch).ordered
+        expect(observer).to receive(:update).with(:start_decode).ordered
+        expect(observer).to receive(:update).with(:end_decode).ordered
+        expect(observer).to receive(:update).with(:start_parse_record, record1).ordered
+        expect(observer).to receive(:update).with(:end_parse_record, record1).ordered
+        expect(observer).to receive(:update).with(:start_parse_record, record2).ordered
+        expect(observer).to receive(:update).with(:end_parse_record, record2).ordered
+        expect(observer).to receive(:update).with(:start_parse_record, record3).ordered
+        expect(observer).to receive(:update).with(:end_parse_record, record3).ordered
+        expect(observer).to receive(:update).with(:end).ordered
+        feed_runner.run()
+      end
+
+      it "notifies the observer with :end last" do
+        feed_runner.run()
+        expect(observer.updates.last).to eq([:end])
+      end
+
+      context "with :io => io" do
+        it "does not build or run the fetcher" do
+          expect(feed.fetcher_builder).not_to receive(:call)
+          feed_runner.run(:io => io)
         end
 
-        context "when providing the :io argument" do
-          it "should not call fetcher_builder, but initialize the parser with the thing we provided to :io"  do
-            expect(fetcher_builder).not_to receive(:call)
-            expect(fetcher).not_to receive(:fetch)
-            feed_runner.run(:io => io)
-          end
+        it "does not notify the observer with :start_fetch or :end_fetch" do
+          expect(observer).not_to receive(:update).with(:start_fetch)
+          expect(observer).not_to receive(:update).with(:end_fetch)
+          feed_runner.run(:io => io)
         end
+      end
 
+      context "without :io" do
         it "should generate a new fetcher via fetcher_builder.call, and then fetch" do
-          expect(fetcher_builder).to receive(:call).and_call_original
-          expect(fetcher).to receive(:fetch).and_return(io)
+          expect(feed.fetcher_builder).to receive(:call).and_call_original
+          expect(fetcher).to receive(:fetch).and_call_original
           feed_runner.run
         end
+
+        it "notifies the observer with :start_fetch, then fetches, then notifies observer with :end_fetch" do
+          expect(observer).to receive(:update).with(:start_fetch).ordered
+          expect(fetcher).to receive(:fetch).and_call_original.ordered
+          expect(observer).to receive(:update).with(:end_fetch).ordered
+          feed_runner.run()
+        end
       end
 
-      context "parsing" do
-        before :each do
-          allow(fetcher).to receive(:fetch).and_return(io)
+      it "calls the feed.parser_block for each for each message data parsed" do
+        expect(feed.parser_block).to receive(:call).with(kind_of(Proc), record1).ordered
+        expect(feed.parser_block).to receive(:call).with(kind_of(Proc), record2).ordered
+        expect(feed.parser_block).to receive(:call).with(kind_of(Proc), record3).ordered
+        feed_runner.run()
+      end
+
+
+      context "when handling record" do
+        let(:records) { [ record1 ] }
+
+        it "notifies observer with :start_parse_record, and the record prior to handling" do
+          expect(observer).to receive(:update).with(:start_parse_record, record1).ordered
+          expect(feed.parser_block).to receive(:call).with(kind_of(Proc), record1).ordered
+          feed_runner.run()
         end
 
-        it "should call the parser_block for each for each message data parsed" do
-          record1 = Threatinator::Record.new('a1')
-          record2 = Threatinator::Record.new('a2')
-          record3 = Threatinator::Record.new('a3')
-          expect(parser).to receive(:run).with(io).and_yield(record1).and_yield(record2).and_yield(record3)
+        it "notifies observer with :end_parse_record, and the record after handling" do
           expect(feed.parser_block).to receive(:call).with(kind_of(Proc), record1).ordered
-          expect(feed.parser_block).to receive(:call).with(kind_of(Proc), record2).ordered
-          expect(feed.parser_block).to receive(:call).with(kind_of(Proc), record3).ordered
-          feed_runner.run
+          expect(observer).to receive(:update).with(:end_parse_record, record1).ordered
+          feed_runner.run()
+        end
+
+        context "when the record is parsed into one or more events" do
+          let(:parser_block) { 
+            lambda do |cep, record| 
+              cep.call do |eb|
+                eb.type = :c2
+                eb.add_ipv4('1.1.1.1')
+              end
+              cep.call do |eb|
+                eb.type = :c2
+                eb.add_ipv4('2.2.2.2')
+              end
+            end
+          }
+
+          it "notifies the observer with (:record_parsed, record, events) for each event" do
+            expect(observer).to receive(:update).with(
+              :record_parsed, record1, satisfy { |events| 
+                expect(events[0].ipv4s).to contain_exactly(build(:ipv4, ipv4: '1.1.1.1'))
+                expect(events[1].ipv4s).to contain_exactly(build(:ipv4, ipv4: '2.2.2.2'))
+              }) 
+
+            feed_runner.run()
+          end
+        end
+
+        context "when no events have been parsed from the record" do
+          let(:parser_block) { 
+            lambda do |cep, record| 
+            end
+          }
+
+          it "notifies the observer with (:record_missed, record) for each event" do
+            expect(observer).to receive(:update).with(:record_missed, record1)
+            feed_runner.run()
+          end
+        end
+
+        context "when a record has been filtered" do
+          let(:filters) { [ lambda { |record| true } ] }
+          it "notifies the observer with (:record_filtered, record)" do
+            expect(observer).to receive(:update).with(:record_filtered, record1)
+            feed_runner.run()
+          end
+        end
+
+        context "when a record has an event that fails to build" do
+          let(:parser_block) { 
+            lambda do |cep, record| 
+              cep.call do |eb|
+                eb.type = :c2
+              end
+              cep.call do |eb|
+                eb.type = :asdf
+              end
+            end
+          }
+
+          it "notifies the observer with (:record_error, record, array_of_errors)" do
+            expect(observer).to receive(:update).with(:record_error, record1, a_collection_containing_exactly(
+              kind_of(Threatinator::Exceptions::EventBuildError)
+            ))
+            feed_runner.run()
+          end
+
+          it "does not notify the observer of any events that may have NOT have errors" do
+            expect(observer).not_to receive(:update).with(:record_parsed, record1, kind_of(Object))
+            feed_runner.run()
+          end
         end
+
       end
+
       context "filtering" do
+        let(:filters) { [ lambda { |record| record.data == "a2" } ] }
+
         before :each do
-          allow(parser).to receive(:run).with(io)
           allow(fetcher).to receive(:fetch).and_return(io)
         end
-        let(:filter) { double("filter") }
-        let(:filter_builders) { [ lambda {filter} ] }
-        it "should not call the parser_block if the data was filtered" do
-          allow(filter).to receive(:filter?)
-          allow(feed_runner).to receive(:_fetch).and_return(io)
-          record1 = Threatinator::Record.new('a1')
-          record2 = Threatinator::Record.new('a2')
-          record3 = Threatinator::Record.new('a3')
-
-          expect(parser).to receive(:run).with(io).and_yield(record1).and_yield(record2).and_yield(record3)
-          expect(filter).to receive(:filter?).with(record1).ordered.and_return(false)
-          expect(feed.parser_block).to receive(:call).with(kind_of(Proc), record1).ordered
 
-          expect(filter).to receive(:filter?).with(record2).ordered.and_return(true)
-          expect(filter).to receive(:filter?).with(record3).ordered.and_return(false)
+        it "only calls the parser_block for data that was not filtered" do
+          expect(feed.parser_block).to receive(:call).with(kind_of(Proc), record1).ordered
           expect(feed.parser_block).to receive(:call).with(kind_of(Proc), record3).ordered
-
-          feed_runner.run
+          feed_runner.run()
         end
       end
 
       context "decoding" do
-        before :each do
-          allow(fetcher).to receive(:fetch).and_return(io)
-        end
-        let(:decoder1) { double("decoder") }
-        let(:decoder2) { double("decoder") }
-        let(:decoder3) { double("decoder") }
-        let(:decoder_builders) { [ lambda {decoder1}, lambda {decoder2}, lambda {decoder3} ] }
-        it "should run through each decoder in the order it was added to the feed" do
-          decoded_io1 = double("decoded_io1")
-          decoded_io2 = double("decoded_io2")
-          decoded_io3 = double("decoded_io3")
-
-          expect(decoder1).to receive(:decode).with(io).and_return(decoded_io1)
-          expect(decoder2).to receive(:decode).with(decoded_io1).and_return(decoded_io2)
-          expect(decoder3).to receive(:decode).with(decoded_io2).and_return(decoded_io3)
-          expect(parser).to receive(:run).with(decoded_io3)
-          feed_runner.run
+        let(:decoded_io1) { create_mock_io('decoded_io1') }
+        let(:decoded_io2) { create_mock_io('decoded_io2') }
+        let(:decoded_io3) { create_mock_io('decoded_io3') }
+        let(:decoder1) { DummyDecoder.new(decoded_io1) }
+        let(:decoder2) { DummyDecoder.new(decoded_io2) }
+        let(:decoder3) { DummyDecoder.new(decoded_io3) }
+        let(:decoders) { [ decoder1, decoder2, decoder3 ] }
+
+        context "without :skip_decoding" do
+          it "notifies the observer with :start_decode, decodes, and then notifies the observer with :end_decode" do
+            expect(observer).to receive(:update).with(:start_decode).ordered
+            expect(decoder1).to receive(:decode).with(io).and_call_original.ordered
+            expect(decoder2).to receive(:decode).with(decoded_io1).and_call_original.ordered
+            expect(decoder3).to receive(:decode).with(decoded_io2).and_call_original.ordered
+            expect(observer).to receive(:update).with(:end_decode).ordered
+            feed_runner.run
+          end
+
+          it "should run through each decoder in the order it was added to the feed" do
+            expect(decoder1).to receive(:decode).with(io).and_call_original
+            expect(decoder2).to receive(:decode).with(decoded_io1).and_call_original
+            expect(decoder3).to receive(:decode).with(decoded_io2).and_call_original
+            expect(parser).to receive(:run).with(decoded_io3)
+            feed_runner.run
+          end
         end
 
-        it "should skip decoding if the :skip_decoding was set to true" do
-          expect(parser).to receive(:run).with(io)
-          feed_runner.run(skip_decoding: true)
+        context "with :skip_decoding => true" do
+          it "does not decode" do
+            expect(observer).not_to receive(:update).with(:start_decode)
+            expect(observer).not_to receive(:update).with(:end_decode)
+            expect(parser).to receive(:run).with(io)
+            feed_runner.run(skip_decoding: true)
+          end
         end
       end
     end

data/spec/threatinator/logger_spec.rb

@@ -0,0 +1,29 @@
+require 'spec_helper'
+require 'threatinator/logger'
+require 'threatinator/config/logger'
+
+describe Threatinator::Logger do
+  before :each do
+    @orig_level = Threatinator::Logger.level
+  end
+
+  after :each do
+    Threatinator::Logger.level = @orig_level
+  end
+
+  describe ".configure_logger(config)" do
+    let(:config) { Threatinator::Config::Logger.new }
+    it "sets the logging level to that specified by config.level" do
+      Threatinator::Logger.level = Threatinator::Logger::Levels::FATAL
+      config.level = "DEBUG"
+      Threatinator::Logger.configure_logger(config)
+      expect(Threatinator::Logger.level).to eq(Threatinator::Logger::Levels::DEBUG)
+    end
+
+    it "warns when an unknown logging level is provided" do
+      config.level = "FOO"
+      expect(Threatinator::Logger.default_logger).to receive(:warn).with(/Ignoring unknown logging level:/)
+      Threatinator::Logger.configure_logger(config)
+    end
+  end
+end

data/spec/threatinator/model/observables/fqdn_collection_spec.rb

@@ -0,0 +1,42 @@
+require 'spec_helper'
+require 'threatinator/model/observables/fqdn_collection'
+
+describe Threatinator::Model::Observables::FqdnCollection do
+  it_behaves_like "a model collection" do
+    def generate_ten_valid_members
+      ret = []
+      1.upto(10) do |i|
+        ret << "domain#{i}.com"
+      end
+      ret
+    end
+
+    def generate_invalid_members
+      [1234, :foobar]
+    end
+  end
+
+  let(:collection) { described_class.new }
+
+  describe "#valid_member?(v)" do
+    context "when provided a valid FQDN string" do
+      it "returns true" do
+        expect(collection.valid_member?("yahoo.com")).to eq(true)
+      end
+    end
+
+    context "when provided an invalid FQDN string" do
+      it "returns false" do
+        pending "doesn't actually validate fqdns, yet"
+        expect(collection.valid_member?("yahoo..com")).to eq(false)
+      end
+    end
+
+    context "when provided something other than a string" do
+      it "returns false" do
+        expect(collection.valid_member?(:asdf)).to eq(false)
+        expect(collection.valid_member?(1234)).to eq(false)
+      end
+    end
+  end
+end