shadowbq-threatinator 0.5.0

data/spec/threatinator/feed_registry_spec.rb

@@ -0,0 +1,198 @@
+require 'spec_helper'
+require 'threatinator/feed_registry'
+require 'threatinator/config/feed_search'
+
+describe Threatinator::FeedRegistry do
+  def generate_feedfile(filename, provider, name, url = "https://foobar/#{provider}/#{name}.data")
+    File.open(filename, "w") do |fio|
+      fio.write <<EOS
+provider "#{provider}"
+name "#{name}"
+fetch_http('#{url}')
+
+parse_eachline(:separator => "\n") do |builder, line|
+end
+EOS
+    end
+  end
+
+  let(:registry) { described_class.new }
+  let(:ten_feeds) { 1.upto(10).map { |i| build(:feed, provider: "prov#{i}", name: "name#{i}") } }
+
+  describe "#clear" do
+    it "should remove all existing registrations" do
+      ten_feeds.each do |feed|
+        registry.register(feed)
+      end
+      expect(registry.count).to eq(10)
+      registry.clear
+      expect(registry.count).to eq(0)
+
+      expect {
+        ten_feeds.each do |feed|
+          registry.register(feed)
+        end
+      }.not_to raise_error
+      expect(registry.count).to eq(10)
+    end
+  end
+
+  describe "#register" do
+    it "should register the provided feed" do
+      feed = build(:feed, provider: "my_provider", name: "my_name")
+      registry.register(feed)
+      expect(registry.get("my_provider", "my_name")).to be(feed)
+    end
+    it "should return the feed that was registered" do
+      feed = build(:feed, provider: "my_provider", name: "my_name")
+      expect(registry.register(feed)).to be(feed)
+    end
+    it "should raise a AlreadyRegisteredError if a feed is already registered with the provider and name" do
+      feed1 = build(:feed, provider: "my_provider", name: "my_name")
+      feed2 = build(:feed, provider: "my_provider", name: "my_name")
+      registry.register(feed1)
+      expect { 
+        registry.register(feed2)
+      }.to raise_error(Threatinator::Exceptions::AlreadyRegisteredError)
+    end
+
+    it "should raise a AlreadyRegisteredError if the same feed is registered twice" do
+      feed = build(:feed, provider: "my_provider", name: "my_name")
+      registry.register(feed)
+      expect { 
+        registry.register(feed)
+      }.to raise_error(Threatinator::Exceptions::AlreadyRegisteredError)
+    end
+  end
+
+  describe "#register_from_file" do
+    let(:feedfile) {FEED_FIXTURES.join("provider1", "feed1.feed").to_s}
+
+    it "should return the feed after parsing the file" do
+      ret = registry.register_from_file(feedfile)
+      expect(ret).to be_a(Threatinator::Feed)
+      expect(ret.provider).to eq("provider1")
+      expect(ret.name).to eq("feed1")
+    end
+
+    it "should have registered the feed" do
+      expect(registry.count).to eq(0)
+      feed = registry.register_from_file(feedfile)
+      expect(registry.count).to eq(1)
+      expect(registry.get(feed.provider, feed.name)).to be(feed)
+    end
+  end
+
+  describe "#each" do
+    it "should enumerate through each reigstered feed" do
+      ten_feeds.each do |feed|
+        registry.register(feed)
+      end
+      found_feeds = []
+      registry.each do |feed|
+        found_feeds << feed
+      end
+      expect(found_feeds).to match_array(ten_feeds)
+    end
+  end
+
+  describe "#count" do
+    it "should return the number of feeds contained within the registry" do
+      expect(registry.count).to eq(0)
+      ten_feeds.each do |feed|
+        registry.register(feed)
+      end
+      expect(registry.count).to eq(10)
+    end
+  end
+
+  describe "#get" do
+    it "should return nil if the provider/name isn't registered" do
+      registry.register(build(:feed, provider: "my_provider", name: "my_name"))
+      expect(registry.get("asdf", "1234")).to be_nil
+    end
+    it "should return the correct feed for the given provider and name" do
+      ten_feeds.each { |feed| registry.register(feed) }
+      feed = build(:feed, provider: "my_provider", name: "my_name")
+      registry.register(feed)
+      expect(registry.get("my_provider", "my_name")).to be(feed)
+    end
+  end
+
+  describe ".build(feed_search_config)" do
+    context "with no feed paths" do
+      let(:feed_search_config) {
+        Threatinator::Config::FeedSearch.new(exclude_default: true, path: [] )
+      }
+      let(:registry) { described_class.build(feed_search_config) }
+
+      it "should not have loaded any feeds" do
+        expect(registry.count).to eq(0)
+      end
+    end
+
+    context "with feed search paths" do
+      let(:feed_search_config) {
+        Threatinator::Config::FeedSearch.new(exclude_default: true, path: [
+          @feed_path1, @feed_path2
+        ] )
+      }
+      before :each do
+        @feed_path1 = Dir.mktmpdir
+        @feed_path2 = Dir.mktmpdir
+      end
+
+      after :each do
+        FileUtils.remove_entry_secure @feed_path1
+        FileUtils.remove_entry_secure @feed_path2
+      end
+
+      it "should load feeds from all of the configured paths" do
+        5.times do |i|
+          generate_feedfile(File.join(@feed_path1, "feed#{i}.feed"), "provider1", "feed#{i}")
+          generate_feedfile(File.join(@feed_path2, "feed#{i}.feed"), "provider2", "feed#{i}")
+        end
+        registry = described_class.build(feed_search_config) 
+        expect(registry.count).to eq(10)
+      end
+
+      it "should ignore files that don't end with .feed" do
+        generate_feedfile(File.join(@feed_path1, "feed1.fee"), "provider1", "feed1")
+        generate_feedfile(File.join(@feed_path1, "feed1.fed"), "provider1", "feed2")
+        generate_feedfile(File.join(@feed_path1, "feed1.rb"), "provider1", "feed3")
+        generate_feedfile(File.join(@feed_path1, "feed1.feed"), "real_provider", "my_feed")
+        registry = described_class.build(feed_search_config) 
+        expect(registry.count).to eq(1)
+        expect(registry.get("real_provider", "my_feed")).to be_a(Threatinator::Feed)
+        expect(registry.get("provider1", "feed1")).to be_nil
+        expect(registry.get("provider1", "feed2")).to be_nil
+        expect(registry.get("provider1", "feed3")).to be_nil
+      end
+
+      it "should recurse subdirectories, loading feeds from there" do
+        level1 = File.join(@feed_path1, "level1")
+        level2 = File.join(@feed_path1, "level1", "level2")
+        level3 = File.join(@feed_path1, "level1", "level2", "level3")
+        FileUtils.mkdir_p level1
+        FileUtils.mkdir_p level2
+        FileUtils.mkdir_p level3
+        generate_feedfile(File.join(level1, "feed1.feed"), "provider1", "feed1")
+        generate_feedfile(File.join(level2, "feed2.feed"), "provider1", "feed2")
+        generate_feedfile(File.join(level3, "feed3.feed"), "provider1", "feed3")
+        registry = described_class.build(feed_search_config) 
+        expect(registry.count).to eq(3)
+      end
+
+      it "should raise an exception if the same feed provider/name combination appears in multiple files" do
+        generate_feedfile(File.join(@feed_path1, "feed1.feed"), "provider1", "feed1")
+        generate_feedfile(File.join(@feed_path1, "feed2.feed"), "provider1", "feed1")
+        expect {
+          described_class.build(feed_search_config) 
+        }.to raise_error(Threatinator::Exceptions::AlreadyRegisteredError)
+      end
+
+    end
+
+
+  end
+end

data/spec/threatinator/feed_runner_spec.rb

@@ -0,0 +1,372 @@
+require 'spec_helper'
+require 'threatinator/feed_runner'
+
+describe Threatinator::FeedRunner do
+  class DummyParser < Threatinator::Parser
+    def initialize(records, opts = {})
+      @records = records
+    end
+
+    def run(io)
+      @records.each do |record|
+        yield(record)
+      end
+    end
+  end
+
+  class DummyFetcher < Threatinator::Fetcher
+    def initialize(io, opts = {})
+      @io = io
+    end
+
+    def fetch
+      return @io
+    end
+  end
+
+  class DummyDecoder < Threatinator::Decoder
+    def initialize(io)
+      @io = io
+    end
+
+    def decode(arg_io)
+      return @io
+    end
+  end
+
+  class TestObserver
+    attr_reader :updates
+    def initialize
+      @updates = []
+    end
+
+    def update(*args)
+      @updates << args
+    end
+  end
+
+  class DummyOutput < Threatinator::Output
+    def handle_event(event); end
+    def finish; end
+  end
+
+  def create_mock_io(name = "io")
+    ret = double(name) 
+    allow(ret).to receive(:close)
+    allow(ret).to receive(:closed?).and_return(false)
+    ret
+  end
+
+  let(:output_formatter ) { DummyOutput.new(nil) }
+  let(:io) { create_mock_io("io") }
+  let(:fetcher) { DummyFetcher.new(io) }
+
+  let(:record1) { Threatinator::Record.new('a1') }
+  let(:record2) { Threatinator::Record.new('a2') }
+  let(:record3) { Threatinator::Record.new('a3') }
+  let(:records) { [ record1, record2, record3 ] }
+  let(:parser) { DummyParser.new(records) }
+  let(:parser_block) { lambda { |*args| } }
+
+  let(:filters) { [] }
+  let(:decoders) { [] }
+
+  let(:feed) {
+    build(:feed, fetcher: fetcher, 
+          parser: parser, filters: filters,
+          decoders: decoders, parser_block: parser_block
+         )
+  }
+
+  describe ".run(feed, output_formatter)" do
+    before :each do
+      @feed_runner = double('feed_runner')
+      allow(described_class).to receive(:new).and_return(@feed_runner)
+      allow(@feed_runner).to receive(:run)
+      described_class.run(feed, output_formatter)
+    end
+
+    let(:feed_runner) {@feed_runner}
+
+    it "initializes a FeedRunner with the given feed and output_formatter" do
+      expect(described_class).to have_received(:new).with(feed,output_formatter)
+    end
+
+    it "calls #run on the feed_runner with an empty hash" do
+      expect(feed_runner).to have_received(:run).with({})
+    end
+  end
+
+  describe ".run(feed, output_formatter, run_opts)" do
+    before :each do
+      @feed_runner = double('feed_runner')
+      allow(described_class).to receive(:new).and_return(@feed_runner)
+      allow(@feed_runner).to receive(:run)
+      described_class.run(feed, output_formatter, run_opts)
+    end
+
+    let(:feed_runner) {@feed_runner}
+    let(:run_opts) { {foo: "bar"} }
+
+    it "initializes a FeedRunner with the given feed and output_formatter" do
+      expect(described_class).to have_received(:new).with(feed,output_formatter)
+    end
+
+    it "calls #run on the feed_runner with an empty hash" do
+      expect(feed_runner).to have_received(:run).with(run_opts)
+    end
+  end
+
+  context "an instance" do
+    let(:feed_runner) { described_class.new(feed, output_formatter) }
+    let(:observer) { TestObserver.new }
+
+    before :each do
+      feed_runner.add_observer(observer)
+      allow(observer).to receive(:update).and_call_original
+    end
+
+    describe "#run" do
+      it "notifies the observer with :start before anything else" do
+        expect(observer.updates.first).to be_nil
+        feed_runner.run()
+        expect(observer.updates.first).to eq([:start])
+      end
+
+      it "fetches, decodes, and then parses records" do
+        expect(observer).to receive(:update).with(:start).ordered
+        expect(observer).to receive(:update).with(:start_fetch).ordered
+        expect(observer).to receive(:update).with(:end_fetch).ordered
+        expect(observer).to receive(:update).with(:start_decode).ordered
+        expect(observer).to receive(:update).with(:end_decode).ordered
+        expect(observer).to receive(:update).with(:start_parse_record, record1).ordered
+        expect(observer).to receive(:update).with(:end_parse_record, record1).ordered
+        expect(observer).to receive(:update).with(:start_parse_record, record2).ordered
+        expect(observer).to receive(:update).with(:end_parse_record, record2).ordered
+        expect(observer).to receive(:update).with(:start_parse_record, record3).ordered
+        expect(observer).to receive(:update).with(:end_parse_record, record3).ordered
+        expect(observer).to receive(:update).with(:end).ordered
+        feed_runner.run()
+      end
+
+      it "notifies the observer with :end last" do
+        feed_runner.run()
+        expect(observer.updates.last).to eq([:end])
+      end
+
+      context "with :io => io" do
+        it "does not build or run the fetcher" do
+          expect(feed.fetcher_builder).not_to receive(:call)
+          feed_runner.run(:io => io)
+        end
+
+        it "does not notify the observer with :start_fetch or :end_fetch" do
+          expect(observer).not_to receive(:update).with(:start_fetch)
+          expect(observer).not_to receive(:update).with(:end_fetch)
+          feed_runner.run(:io => io)
+        end
+      end
+
+      context "without :io" do
+        it "should generate a new fetcher via fetcher_builder.call, and then fetch" do
+          expect(feed.fetcher_builder).to receive(:call).and_call_original
+          expect(fetcher).to receive(:fetch).and_call_original
+          feed_runner.run
+        end
+
+        it "notifies the observer with :start_fetch, then fetches, then notifies observer with :end_fetch" do
+          expect(observer).to receive(:update).with(:start_fetch).ordered
+          expect(fetcher).to receive(:fetch).and_call_original.ordered
+          expect(observer).to receive(:update).with(:end_fetch).ordered
+          feed_runner.run()
+        end
+      end
+
+      it "calls the feed.parser_block for each for each message data parsed" do
+        expect(feed.parser_block).to receive(:call).with(kind_of(Proc), record1).ordered
+        expect(feed.parser_block).to receive(:call).with(kind_of(Proc), record2).ordered
+        expect(feed.parser_block).to receive(:call).with(kind_of(Proc), record3).ordered
+        feed_runner.run()
+      end
+
+
+      context "when handling record" do
+        let(:records) { [ record1 ] }
+
+        it "notifies observer with :start_parse_record, and the record prior to handling" do
+          expect(observer).to receive(:update).with(:start_parse_record, record1).ordered
+          expect(feed.parser_block).to receive(:call).with(kind_of(Proc), record1).ordered
+          feed_runner.run()
+        end
+
+        it "notifies observer with :end_parse_record, and the record after handling" do
+          expect(feed.parser_block).to receive(:call).with(kind_of(Proc), record1).ordered
+          expect(observer).to receive(:update).with(:end_parse_record, record1).ordered
+          feed_runner.run()
+        end
+
+        context "when the record is parsed into one or more events" do
+          let(:parser_block) { 
+            lambda do |cep, record| 
+              cep.call do |eb|
+                eb.type = :c2
+                eb.add_ipv4('1.1.1.1')
+              end
+              cep.call do |eb|
+                eb.type = :c2
+                eb.add_ipv4('2.2.2.2')
+              end
+            end
+          }
+
+          it "notifies the observer with (:record_parsed, record, events) for each event" do
+            expect(observer).to receive(:update).with(
+              :record_parsed, record1, satisfy { |events| 
+                expect(events[0].ipv4s).to contain_exactly(build(:ipv4, ipv4: '1.1.1.1'))
+                expect(events[1].ipv4s).to contain_exactly(build(:ipv4, ipv4: '2.2.2.2'))
+              }) 
+
+            feed_runner.run()
+          end
+        end
+
+        context "when no events have been parsed from the record" do
+          let(:parser_block) { 
+            lambda do |cep, record| 
+            end
+          }
+
+          it "notifies the observer with (:record_missed, record) for each event" do
+            expect(observer).to receive(:update).with(:record_missed, record1)
+            feed_runner.run()
+          end
+        end
+
+        context "when a record has been filtered" do
+          let(:filters) { [ lambda { |record| true } ] }
+          it "notifies the observer with (:record_filtered, record)" do
+            expect(observer).to receive(:update).with(:record_filtered, record1)
+            feed_runner.run()
+          end
+        end
+
+        context "when a record has an event that fails to build" do
+          let(:parser_block) { 
+            lambda do |cep, record| 
+              cep.call do |eb|
+                eb.type = :c2
+              end
+              cep.call do |eb|
+                eb.type = :asdf
+              end
+            end
+          }
+
+          it "notifies the observer with (:record_error, record, array_of_errors)" do
+            expect(observer).to receive(:update).with(:record_error, record1, a_collection_containing_exactly(
+              kind_of(Threatinator::Exceptions::EventBuildError)
+            ))
+            feed_runner.run()
+          end
+
+          it "does not notify the observer of any events that may have NOT have errors" do
+            expect(observer).not_to receive(:update).with(:record_parsed, record1, kind_of(Object))
+            feed_runner.run()
+          end
+        end
+
+      end
+
+      context "filtering" do
+        let(:filters) { [ lambda { |record| record.data == "a2" } ] }
+
+        before :each do
+          allow(fetcher).to receive(:fetch).and_return(io)
+        end
+
+        it "only calls the parser_block for data that was not filtered" do
+          expect(feed.parser_block).to receive(:call).with(kind_of(Proc), record1).ordered
+          expect(feed.parser_block).to receive(:call).with(kind_of(Proc), record3).ordered
+          feed_runner.run()
+        end
+      end
+
+      context "cleanup" do
+        it "closes the IO provided via :io" do
+          expect(io).to receive(:closed?).and_return(false)
+          expect(io).to receive(:close)
+          feed_runner.run(:io => io)
+        end
+
+        it "closes the IO returned by the fetcher" do
+          allow(fetcher).to receive(:fetch).and_return(io)
+          expect(io).to receive(:closed?).and_return(false)
+          expect(io).to receive(:close)
+          feed_runner.run
+        end
+
+        it "calls output_formatter.finish" do
+          expect(output_formatter).to receive(:finish)
+          feed_runner.run
+        end
+
+        context "when handling a chain of IOs created by decoders" do
+          let(:decoded_io1) { create_mock_io('decoded_io1') }
+          let(:decoded_io2) { create_mock_io('decoded_io2') }
+          let(:decoded_io3) { create_mock_io('decoded_io3') }
+          let(:decoder1) { DummyDecoder.new(decoded_io1) }
+          let(:decoder2) { DummyDecoder.new(decoded_io2) }
+          let(:decoder3) { DummyDecoder.new(decoded_io3) }
+          let(:decoders) { [ decoder1, decoder2, decoder3 ] }
+          it "closes each decoded IO at the end" do
+            expect(decoded_io1).to receive(:closed?).and_return(false)
+            expect(decoded_io1).to receive(:close)
+            expect(decoded_io2).to receive(:closed?).and_return(false)
+            expect(decoded_io2).to receive(:close)
+            expect(decoded_io3).to receive(:closed?).and_return(false)
+            expect(decoded_io3).to receive(:close)
+            feed_runner.run
+          end
+        end
+      end
+
+      context "decoding" do
+        let(:decoded_io1) { create_mock_io('decoded_io1') }
+        let(:decoded_io2) { create_mock_io('decoded_io2') }
+        let(:decoded_io3) { create_mock_io('decoded_io3') }
+        let(:decoder1) { DummyDecoder.new(decoded_io1) }
+        let(:decoder2) { DummyDecoder.new(decoded_io2) }
+        let(:decoder3) { DummyDecoder.new(decoded_io3) }
+        let(:decoders) { [ decoder1, decoder2, decoder3 ] }
+
+        context "without :skip_decoding" do
+          it "notifies the observer with :start_decode, decodes, and then notifies the observer with :end_decode" do
+            expect(observer).to receive(:update).with(:start_decode).ordered
+            expect(decoder1).to receive(:decode).with(io).and_call_original.ordered
+            expect(decoder2).to receive(:decode).with(decoded_io1).and_call_original.ordered
+            expect(decoder3).to receive(:decode).with(decoded_io2).and_call_original.ordered
+            expect(observer).to receive(:update).with(:end_decode).ordered
+            feed_runner.run
+          end
+
+          it "should run through each decoder in the order it was added to the feed" do
+            expect(decoder1).to receive(:decode).with(io).and_call_original
+            expect(decoder2).to receive(:decode).with(decoded_io1).and_call_original
+            expect(decoder3).to receive(:decode).with(decoded_io2).and_call_original
+            expect(parser).to receive(:run).with(decoded_io3)
+            feed_runner.run
+          end
+        end
+
+        context "with :skip_decoding => true" do
+          it "does not decode" do
+            expect(observer).not_to receive(:update).with(:start_decode)
+            expect(observer).not_to receive(:update).with(:end_decode)
+            expect(parser).to receive(:run).with(io)
+            feed_runner.run(skip_decoding: true)
+          end
+        end
+      end
+    end
+  end
+end