threatinator 0.1.1

data/spec/threatinator/feed_registry_spec.rb

@@ -0,0 +1,198 @@
+require 'spec_helper'
+require 'threatinator/feed_registry'
+require 'threatinator/config/feed_search'
+
+describe Threatinator::FeedRegistry do
+  def generate_feedfile(filename, provider, name, url = "https://foobar/#{provider}/#{name}.data")
+    File.open(filename, "w") do |fio|
+      fio.write <<EOS
+provider "#{provider}"
+name "#{name}"
+fetch_http('#{url}')
+
+parse_eachline(:separator => "\n") do |builder, line|
+end
+EOS
+    end
+  end
+
+  let(:registry) { described_class.new }
+  let(:ten_feeds) { 1.upto(10).map { |i| build(:feed, provider: "prov#{i}", name: "name#{i}") } }
+
+  describe "#clear" do
+    it "should remove all existing registrations" do
+      ten_feeds.each do |feed|
+        registry.register(feed)
+      end
+      expect(registry.count).to eq(10)
+      registry.clear
+      expect(registry.count).to eq(0)
+
+      expect {
+        ten_feeds.each do |feed|
+          registry.register(feed)
+        end
+      }.not_to raise_error
+      expect(registry.count).to eq(10)
+    end
+  end
+
+  describe "#register" do
+    it "should register the provided feed" do
+      feed = build(:feed, provider: "my_provider", name: "my_name")
+      registry.register(feed)
+      expect(registry.get("my_provider", "my_name")).to be(feed)
+    end
+    it "should return the feed that was registered" do
+      feed = build(:feed, provider: "my_provider", name: "my_name")
+      expect(registry.register(feed)).to be(feed)
+    end
+    it "should raise a AlreadyRegisteredError if a feed is already registered with the provider and name" do
+      feed1 = build(:feed, provider: "my_provider", name: "my_name")
+      feed2 = build(:feed, provider: "my_provider", name: "my_name")
+      registry.register(feed1)
+      expect { 
+        registry.register(feed2)
+      }.to raise_error(Threatinator::Exceptions::AlreadyRegisteredError)
+    end
+
+    it "should raise a AlreadyRegisteredError if the same feed is registered twice" do
+      feed = build(:feed, provider: "my_provider", name: "my_name")
+      registry.register(feed)
+      expect { 
+        registry.register(feed)
+      }.to raise_error(Threatinator::Exceptions::AlreadyRegisteredError)
+    end
+  end
+
+  describe "#register_from_file" do
+    let(:feedfile) {FEED_FIXTURES.join("provider1", "feed1.feed").to_s}
+
+    it "should return the feed after parsing the file" do
+      ret = registry.register_from_file(feedfile)
+      expect(ret).to be_a(Threatinator::Feed)
+      expect(ret.provider).to eq("provider1")
+      expect(ret.name).to eq("feed1")
+    end
+
+    it "should have registered the feed" do
+      expect(registry.count).to eq(0)
+      feed = registry.register_from_file(feedfile)
+      expect(registry.count).to eq(1)
+      expect(registry.get(feed.provider, feed.name)).to be(feed)
+    end
+  end
+
+  describe "#each" do
+    it "should enumerate through each reigstered feed" do
+      ten_feeds.each do |feed|
+        registry.register(feed)
+      end
+      found_feeds = []
+      registry.each do |feed|
+        found_feeds << feed
+      end
+      expect(found_feeds).to match_array(ten_feeds)
+    end
+  end
+
+  describe "#count" do
+    it "should return the number of feeds contained within the registry" do
+      expect(registry.count).to eq(0)
+      ten_feeds.each do |feed|
+        registry.register(feed)
+      end
+      expect(registry.count).to eq(10)
+    end
+  end
+
+  describe "#get" do
+    it "should return nil if the provider/name isn't registered" do
+      registry.register(build(:feed, provider: "my_provider", name: "my_name"))
+      expect(registry.get("asdf", "1234")).to be_nil
+    end
+    it "should return the correct feed for the given provider and name" do
+      ten_feeds.each { |feed| registry.register(feed) }
+      feed = build(:feed, provider: "my_provider", name: "my_name")
+      registry.register(feed)
+      expect(registry.get("my_provider", "my_name")).to be(feed)
+    end
+  end
+
+  describe ".build(feed_search_config)" do
+    context "with no feed paths" do
+      let(:feed_search_config) {
+        Threatinator::Config::FeedSearch.new(exclude_default: true, path: [] )
+      }
+      let(:registry) { described_class.build(feed_search_config) }
+
+      it "should not have loaded any feeds" do
+        expect(registry.count).to eq(0)
+      end
+    end
+
+    context "with feed search paths" do
+      let(:feed_search_config) {
+        Threatinator::Config::FeedSearch.new(exclude_default: true, path: [
+          @feed_path1, @feed_path2
+        ] )
+      }
+      before :each do
+        @feed_path1 = Dir.mktmpdir
+        @feed_path2 = Dir.mktmpdir
+      end
+
+      after :each do
+        FileUtils.remove_entry_secure @feed_path1
+        FileUtils.remove_entry_secure @feed_path2
+      end
+
+      it "should load feeds from all of the configured paths" do
+        5.times do |i|
+          generate_feedfile(File.join(@feed_path1, "feed#{i}.feed"), "provider1", "feed#{i}")
+          generate_feedfile(File.join(@feed_path2, "feed#{i}.feed"), "provider2", "feed#{i}")
+        end
+        registry = described_class.build(feed_search_config) 
+        expect(registry.count).to eq(10)
+      end
+
+      it "should ignore files that don't end with .feed" do
+        generate_feedfile(File.join(@feed_path1, "feed1.fee"), "provider1", "feed1")
+        generate_feedfile(File.join(@feed_path1, "feed1.fed"), "provider1", "feed2")
+        generate_feedfile(File.join(@feed_path1, "feed1.rb"), "provider1", "feed3")
+        generate_feedfile(File.join(@feed_path1, "feed1.feed"), "real_provider", "my_feed")
+        registry = described_class.build(feed_search_config) 
+        expect(registry.count).to eq(1)
+        expect(registry.get("real_provider", "my_feed")).to be_a(Threatinator::Feed)
+        expect(registry.get("provider1", "feed1")).to be_nil
+        expect(registry.get("provider1", "feed2")).to be_nil
+        expect(registry.get("provider1", "feed3")).to be_nil
+      end
+
+      it "should recurse subdirectories, loading feeds from there" do
+        level1 = File.join(@feed_path1, "level1")
+        level2 = File.join(@feed_path1, "level1", "level2")
+        level3 = File.join(@feed_path1, "level1", "level2", "level3")
+        FileUtils.mkdir_p level1
+        FileUtils.mkdir_p level2
+        FileUtils.mkdir_p level3
+        generate_feedfile(File.join(level1, "feed1.feed"), "provider1", "feed1")
+        generate_feedfile(File.join(level2, "feed2.feed"), "provider1", "feed2")
+        generate_feedfile(File.join(level3, "feed3.feed"), "provider1", "feed3")
+        registry = described_class.build(feed_search_config) 
+        expect(registry.count).to eq(3)
+      end
+
+      it "should raise an exception if the same feed provider/name combination appears in multiple files" do
+        generate_feedfile(File.join(@feed_path1, "feed1.feed"), "provider1", "feed1")
+        generate_feedfile(File.join(@feed_path1, "feed2.feed"), "provider1", "feed1")
+        expect {
+          described_class.build(feed_search_config) 
+        }.to raise_error(Threatinator::Exceptions::AlreadyRegisteredError)
+      end
+
+    end
+
+
+  end
+end

data/spec/threatinator/feed_runner_spec.rb

@@ -0,0 +1,155 @@
+require 'spec_helper'
+require 'threatinator/feed_runner'
+
+describe Threatinator::FeedRunner do
+  let(:output_formatter ) { double("formatter") }
+  let(:fetcher) { double("fetcher") }
+  let(:fetcher_builder) { lambda { fetcher } }
+
+  let(:io) { double("io") }
+  let(:parser) { double("parser") }
+  let(:parser_builder) { lambda { parser} }
+
+  let(:filter_builders) { [] }
+  let(:decoder_builders) { [] }
+
+  let(:feed) {
+    build(:feed, fetcher_builder: fetcher_builder, 
+          parser_builder: parser_builder, filter_builders: filter_builders,
+          decoder_builders: decoder_builders
+         )
+  }
+
+  describe ".run(feed, output_formatter)" do
+    before :each do
+      @feed_runner = double('feed_runner')
+      allow(described_class).to receive(:new).and_return(@feed_runner)
+      allow(@feed_runner).to receive(:run)
+      described_class.run(feed, output_formatter)
+    end
+
+    let(:feed_runner) {@feed_runner}
+
+    it "initializes a FeedRunner with the given feed and output_formatter" do
+      expect(described_class).to have_received(:new).with(feed,output_formatter)
+    end
+
+    it "calls #run on the feed_runner with an empty hash" do
+      expect(feed_runner).to have_received(:run).with({})
+    end
+  end
+
+  describe ".run(feed, output_formatter, run_opts)" do
+    before :each do
+      @feed_runner = double('feed_runner')
+      allow(described_class).to receive(:new).and_return(@feed_runner)
+      allow(@feed_runner).to receive(:run)
+      described_class.run(feed, output_formatter, run_opts)
+    end
+
+    let(:feed_runner) {@feed_runner}
+    let(:run_opts) { {foo: "bar"} }
+
+    it "initializes a FeedRunner with the given feed and output_formatter" do
+      expect(described_class).to have_received(:new).with(feed,output_formatter)
+    end
+
+    it "calls #run on the feed_runner with an empty hash" do
+      expect(feed_runner).to have_received(:run).with(run_opts)
+    end
+  end
+
+  context "an instance" do
+    let(:feed_runner) { described_class.new(feed, output_formatter) }
+
+    describe "#run" do
+      context "fetching data" do
+        before :each do
+          allow(parser).to receive(:run).with(io)
+        end
+
+        context "when providing the :io argument" do
+          it "should not call fetcher_builder, but initialize the parser with the thing we provided to :io"  do
+            expect(fetcher_builder).not_to receive(:call)
+            expect(fetcher).not_to receive(:fetch)
+            feed_runner.run(:io => io)
+          end
+        end
+
+        it "should generate a new fetcher via fetcher_builder.call, and then fetch" do
+          expect(fetcher_builder).to receive(:call).and_call_original
+          expect(fetcher).to receive(:fetch).and_return(io)
+          feed_runner.run
+        end
+      end
+
+      context "parsing" do
+        before :each do
+          allow(fetcher).to receive(:fetch).and_return(io)
+        end
+
+        it "should call the parser_block for each for each message data parsed" do
+          record1 = Threatinator::Record.new('a1')
+          record2 = Threatinator::Record.new('a2')
+          record3 = Threatinator::Record.new('a3')
+          expect(parser).to receive(:run).with(io).and_yield(record1).and_yield(record2).and_yield(record3)
+          expect(feed.parser_block).to receive(:call).with(kind_of(Proc), record1).ordered
+          expect(feed.parser_block).to receive(:call).with(kind_of(Proc), record2).ordered
+          expect(feed.parser_block).to receive(:call).with(kind_of(Proc), record3).ordered
+          feed_runner.run
+        end
+      end
+      context "filtering" do
+        before :each do
+          allow(parser).to receive(:run).with(io)
+          allow(fetcher).to receive(:fetch).and_return(io)
+        end
+        let(:filter) { double("filter") }
+        let(:filter_builders) { [ lambda {filter} ] }
+        it "should not call the parser_block if the data was filtered" do
+          allow(filter).to receive(:filter?)
+          allow(feed_runner).to receive(:_fetch).and_return(io)
+          record1 = Threatinator::Record.new('a1')
+          record2 = Threatinator::Record.new('a2')
+          record3 = Threatinator::Record.new('a3')
+
+          expect(parser).to receive(:run).with(io).and_yield(record1).and_yield(record2).and_yield(record3)
+          expect(filter).to receive(:filter?).with(record1).ordered.and_return(false)
+          expect(feed.parser_block).to receive(:call).with(kind_of(Proc), record1).ordered
+
+          expect(filter).to receive(:filter?).with(record2).ordered.and_return(true)
+          expect(filter).to receive(:filter?).with(record3).ordered.and_return(false)
+          expect(feed.parser_block).to receive(:call).with(kind_of(Proc), record3).ordered
+
+          feed_runner.run
+        end
+      end
+
+      context "decoding" do
+        before :each do
+          allow(fetcher).to receive(:fetch).and_return(io)
+        end
+        let(:decoder1) { double("decoder") }
+        let(:decoder2) { double("decoder") }
+        let(:decoder3) { double("decoder") }
+        let(:decoder_builders) { [ lambda {decoder1}, lambda {decoder2}, lambda {decoder3} ] }
+        it "should run through each decoder in the order it was added to the feed" do
+          decoded_io1 = double("decoded_io1")
+          decoded_io2 = double("decoded_io2")
+          decoded_io3 = double("decoded_io3")
+
+          expect(decoder1).to receive(:decode).with(io).and_return(decoded_io1)
+          expect(decoder2).to receive(:decode).with(decoded_io1).and_return(decoded_io2)
+          expect(decoder3).to receive(:decode).with(decoded_io2).and_return(decoded_io3)
+          expect(parser).to receive(:run).with(decoded_io3)
+          feed_runner.run
+        end
+
+        it "should skip decoding if the :skip_decoding was set to true" do
+          expect(parser).to receive(:run).with(io)
+          feed_runner.run(skip_decoding: true)
+        end
+      end
+    end
+  end
+end

data/spec/threatinator/feed_spec.rb

@@ -0,0 +1,169 @@
+require 'spec_helper'
+require 'threatinator/feed'
+
+describe Threatinator::Feed do
+  let (:provider) { 'FakeSecureCo' }
+  let (:name) { 'MaliciousDataFeed' }
+  let(:fetcher_io) { double("io") }
+  let(:parser_block) { lambda {}  }
+  let(:filter_builders) { [] }
+  let(:decoder_builders) { [] }
+  let(:fetcher_builder) { lambda { FeedSpec::Fetcher.new({}) }  }
+  let(:parser_builder) { lambda { FeedSpec::Parser.new({}) { } }  }
+
+  let(:feed_opts) { 
+    {
+      :provider => provider, 
+      :name => name,
+      :parser_block => parser_block,
+      :fetcher_builder => fetcher_builder,
+      :parser_builder => parser_builder,
+      :filter_builders => filter_builders,
+      :decoder_builders => filter_builders,
+    }
+  }
+
+  shared_examples_for "a field with an invalid value" do |value|
+    it "should raise InvalidAttributeError if it is a #{value.class}" do
+      expect { 
+        described_class.new(feed_opts.merge(field => value)) 
+      }.to raise_error(Threatinator::Exceptions::InvalidAttributeError)
+    end
+  end
+
+  shared_examples_for "a field with a valid value" do |value|
+    it "should not raise any errors when set to a #{value.class}" do
+      expect { 
+        described_class.new(feed_opts.merge(field => value)) 
+      }.not_to raise_error
+    end
+  end
+
+  shared_examples_for "a field that is required" do
+    it "is required" do
+      feed_opts.delete(field)
+      expect { described_class.new(feed_opts) }.to raise_error(Threatinator::Exceptions::InvalidAttributeError)
+    end
+  end
+
+  shared_examples_for "a field with a default" do |value|
+    it "should default to #{value.inspect}" do
+      feed_opts.delete(field)
+      feed = described_class.new(feed_opts)
+      expect(feed.send(field)).to eq(value)
+    end
+  end
+
+  shared_examples_for "a field that is immutable" do
+    it "should be immutable" do
+      feed = described_class.new(feed_opts) 
+      expect(feed.send(field)).not_to be(feed.send(field))
+    end
+  end
+
+  describe ":provider" do
+    let(:field) { :provider } 
+    include_examples "a field that is required"
+    include_examples "a field that is immutable"
+    include_examples "a field with a valid value", "asdf"
+    include_examples "a field with an invalid value", 1234
+    include_examples "a field with an invalid value", :asdf
+    include_examples "a field with an invalid value", []
+    include_examples "a field with an invalid value", {}
+  end
+
+  describe ":name" do
+    let(:field) { :name } 
+    include_examples "a field that is required", :name
+    include_examples "a field that is immutable"
+    include_examples "a field with a valid value", "asdf"
+    include_examples "a field with an invalid value", 1234
+    include_examples "a field with an invalid value", :asdf
+    include_examples "a field with an invalid value", []
+    include_examples "a field with an invalid value", {}
+  end
+
+  describe ":fetcher_builder" do
+    let(:field) { :fetcher_builder} 
+    include_examples "a field that is required", :fetcher_builder
+    include_examples "a field with a valid value", lambda { } 
+    include_examples "a field with an invalid value", Class.new
+    include_examples "a field with an invalid value", 1234
+    include_examples "a field with an invalid value", :asdf
+    include_examples "a field with an invalid value", []
+    include_examples "a field with an invalid value", {}
+  end
+
+  describe ":parser_builder" do
+    let(:field) { :parser_builder } 
+    include_examples "a field that is required", :parser_builder
+    include_examples "a field with a valid value", lambda { } 
+    include_examples "a field with an invalid value", Class.new
+    include_examples "a field with an invalid value", 1234
+    include_examples "a field with an invalid value", :asdf
+    include_examples "a field with an invalid value", []
+    include_examples "a field with an invalid value", {}
+  end
+
+  describe ":filter_builders" do
+    let(:field) { :filter_builders} 
+    include_examples "a field with a default", []
+    include_examples "a field with a valid value", []
+    include_examples "a field with a valid value", [ lambda { }  ]
+    include_examples "a field that is immutable"
+    include_examples "a field with an invalid value", Class.new
+    include_examples "a field with an invalid value", 1234
+    include_examples "a field with an invalid value", :asdf
+    include_examples "a field with an invalid value", [1,2,3]
+    include_examples "a field with an invalid value", {}
+  end
+
+  describe ":decoder_builders" do
+    let(:field) { :decoder_builders} 
+    include_examples "a field with a default", []
+    include_examples "a field with a valid value", []
+    include_examples "a field with a valid value", [ lambda { }  ]
+    include_examples "a field that is immutable"
+    include_examples "a field with an invalid value", Class.new
+    include_examples "a field with an invalid value", 1234
+    include_examples "a field with an invalid value", :asdf
+    include_examples "a field with an invalid value", [1,2,3]
+    include_examples "a field with an invalid value", {}
+  end
+
+  describe ":parser_block" do
+    let(:field) { :parser_block } 
+    include_examples "a field that is required", :parser_block
+    include_examples "a field with a valid value", lambda {}
+    include_examples "a field with an invalid value", 1234
+    include_examples "a field with an invalid value", :asdf
+    include_examples "a field with an invalid value", []
+  end
+
+  context "when initialized with required fields" do
+    let (:feed) do 
+      described_class.new(feed_opts) 
+    end
+
+    describe "#name" do
+      it "should return the name" do
+        expect(feed.name).to eq(name)
+      end
+    end
+
+    describe "#provider" do
+      it "should return the provider" do
+        expect(feed.provider).to eq(provider)
+      end
+    end
+
+    describe "#parser_block" do
+      it "should return the parser_block" do
+        expect(feed.parser_block).to eq(parser_block)
+      end
+    end
+
+  end
+end
+
+