shadowbq-threatinator 0.5.0

data/spec/support/shared/record.rb

@@ -0,0 +1,111 @@
+shared_examples_for 'a record when compared to an identically configured record' do
+  let(:record1) { described_class.new(data, opts) }
+  let(:record2) { described_class.new(data2, opts2) }
+  specify "record1 should == record2" do
+    expect(record1).to be == record2
+  end
+  specify "record2 should == record1" do
+    expect(record2).to be == record1
+  end
+  specify "record1 should eql? record2" do
+    expect(record1).to eql record2
+  end
+  specify "record2 should eql? record1" do
+    expect(record2).to eql record1
+  end
+end
+
+shared_examples_for 'a record when compared to a differently configured record' do
+  let(:record1) { described_class.new(data, opts) }
+  let(:record2) { described_class.new(data2, opts2) }
+  specify "record1 should not == record2" do
+    expect(record1).not_to be == record2
+  end
+  specify "record2 should not == record1" do
+    expect(record2).not_to be == record1
+  end
+  specify "record1 should not eql? record2" do
+    expect(record1).not_to eql record2
+  end
+  specify "record2 should not eql? record1" do
+    expect(record2).not_to eql record1
+  end
+end
+
+shared_examples_for 'a record' do
+  let(:record) { described_class.new(data, opts) }
+  context "when compared to itself" do
+    it_should_behave_like 'a record when compared to an identically configured record' do
+      let(:data2) { Marshal.load(Marshal.dump(data)) }
+      let(:opts2) { Marshal.load(Marshal.dump(opts)) }
+    end
+  end
+  describe "initialization options" do
+    describe "data" do
+      it "should be accessible via #data" do
+        expect(record.data).to eq(data)
+      end
+    end
+    describe ":line_number" do
+      context "when not set" do
+        it "should default to nil" do
+          expect(record.line_number).to be_nil
+        end
+      end
+      context "when set" do
+        before :each do
+          opts[:line_number] = 1234
+        end
+        it "should be readable via #line_number" do
+          expect(record.line_number).to eq(1234)
+        end
+
+        it_should_behave_like 'a record when compared to an identically configured record' do
+          let(:data2) { Marshal.load(Marshal.dump(data)) }
+          let(:opts2) { Marshal.load(Marshal.dump(opts)) }
+        end
+      end
+    end
+    describe ":pos_start" do
+      context "when not set" do
+        it "should default to nil" do
+          expect(record.pos_start).to be_nil
+        end
+      end
+      context "when set" do
+        before :each do
+          opts[:pos_start] = 999
+        end
+        it "should be readable via #pos_start" do
+          expect(record.pos_start).to eq(999)
+        end
+
+        it_should_behave_like 'a record when compared to an identically configured record' do
+          let(:data2) { Marshal.load(Marshal.dump(data)) }
+          let(:opts2) { Marshal.load(Marshal.dump(opts)) }
+        end
+      end
+    end
+    describe ":pos_end" do
+      context "when not set" do
+        it "should default to nil" do
+          expect(record.pos_end).to be_nil
+        end
+      end
+      context "when set" do
+        before :each do
+          opts[:pos_end] = 555
+        end
+
+        it "should be readable via #pos_end" do
+          expect(record.pos_end).to eq(555)
+        end
+
+        it_should_behave_like 'a record when compared to an identically configured record' do
+          let(:data2) { Marshal.load(Marshal.dump(data)) }
+          let(:opts2) { Marshal.load(Marshal.dump(opts)) }
+        end
+      end
+    end
+  end
+end

data/spec/threatinator/actions/list/action_spec.rb

@@ -0,0 +1,148 @@
+require 'spec_helper'
+require 'threatinator/actions/list/action'
+require 'threatinator/actions/list/config'
+require 'multi_json'
+
+describe Threatinator::Actions::List::Action do
+  let(:feed_registry) { build(:feed_registry) }
+  let(:config) { Threatinator::Actions::List::Config.new }
+  let(:action) { described_class.new(feed_registry, config) }
+
+  shared_examples_for 'table output' do
+    describe "the header row, header separator, and footer separator" do
+      it "should vary the width of 'provider' based on the longest provider name" do
+
+        1.upto(3) do |i|
+          feed_registry.register(build(:feed, :http, url: "http://#{i}", name:i.to_s, provider: ('A' * (10 * i)), event_types: [:unknown]))
+        end
+
+        output = temp_stdout do
+          action.exec
+        end
+        lines = output.lines.to_a
+
+        expect(lines[0]).to eq("provider                        name  type  link/path event_types\n")
+        expect(lines[1]).to eq("------------------------------  ----  ----  --------- -----------\n")
+        expect(lines[-2]).to eq("------------------------------  ----  ----  --------- -----------\n")
+      end
+
+      it "should vary the width of 'name' based on the longest feed name" do
+        1.upto(3) do |i|
+          feed_registry.register(build(:feed, :http, url: "http://#{i}", provider:i.to_s, name: ('A' * (10 * i)), event_types: [:unknown]))
+        end
+
+        output = temp_stdout do
+          action.exec
+        end
+        lines = output.lines.to_a
+        expect(lines[0]).to eq("provider  name                            type  link/path event_types\n")
+        expect(lines[1]).to eq("--------  ------------------------------  ----  --------- -----------\n")
+        expect(lines[-2]).to eq("--------  ------------------------------  ----  --------- -----------\n")
+      end
+
+      it "should vary the width of 'link/path' based on the longest link name" do
+        1.upto(3) do |i|
+          feed_registry.register(build(:feed, :http, url: 'http://' + ('A' * (i * 10)), provider:i.to_s,name:i.to_s, event_types: [:unknown]))
+        end
+
+        output = temp_stdout do
+          action.exec
+        end
+        lines = output.lines.to_a
+        expect(lines[0]).to eq("provider  name  type  link/path                             event_types\n")
+        expect(lines[1]).to eq("--------  ----  ----  ------------------------------------- -----------\n")
+        expect(lines[-2]).to eq("--------  ----  ----  ------------------------------------- -----------\n")
+      end
+    end
+
+    describe "the list of feeds" do
+      it "should be sorted by provider name and then feed name" do
+        feed_registry.register(build(:feed, provider: 'provider_b', name: 'feed_c', event_types: [:unknown] ))
+        feed_registry.register(build(:feed, provider: 'provider_a', name: 'feed_d', event_types: [:unknown] ))
+        feed_registry.register(build(:feed, provider: 'provider_a', name: 'feed_a', event_types: [:unknown] ))
+        feed_registry.register(build(:feed, provider: 'provider_b', name: 'feed_d', event_types: [:unknown] ))
+        feed_registry.register(build(:feed, provider: 'provider_a', name: 'feed_c', event_types: [:unknown] ))
+        feed_registry.register(build(:feed, provider: 'provider_b', name: 'feed_a', event_types: [:unknown] ))
+        feed_registry.register(build(:feed, provider: 'provider_b', name: 'feed_b', event_types: [:unknown] ))
+        feed_registry.register(build(:feed, provider: 'provider_a', name: 'feed_b', event_types: [:unknown] ))
+
+        output = temp_stdout do
+          action.exec
+        end
+        lines = output.lines.to_a
+        expect(lines[2]).to match(/^provider_a  feed_a  .*$/)
+        expect(lines[3]).to match(/^provider_a  feed_b  .*$/)
+        expect(lines[4]).to match(/^provider_a  feed_c  .*$/)
+        expect(lines[5]).to match(/^provider_a  feed_d  .*$/)
+        expect(lines[6]).to match(/^provider_b  feed_a  .*$/)
+        expect(lines[7]).to match(/^provider_b  feed_b  .*$/)
+        expect(lines[8]).to match(/^provider_b  feed_c  .*$/)
+        expect(lines[9]).to match(/^provider_b  feed_d  .*$/)
+      end
+    end
+
+    describe "the footer" do
+      it "should indicate the number of feeds" do
+        20.times do |i|
+          feed_registry.register(build(:feed))
+        end
+        output = temp_stdout do
+          action.exec
+        end
+        lines = output.lines.to_a
+        expect(lines[-1]).to eq("Total: 20\n")
+      end
+    end
+  end
+
+  shared_examples_for 'json output' do
+    describe "the output" do
+      it "should be sorted by provider name and then feed name" do
+        feed_registry.register(build(:feed, provider: 'provider_b', name: 'feed_c', event_types: [:unknown] ))
+        feed_registry.register(build(:feed, provider: 'provider_a', name: 'feed_d', event_types: [:unknown] ))
+        feed_registry.register(build(:feed, provider: 'provider_a', name: 'feed_a', event_types: [:unknown] ))
+        feed_registry.register(build(:feed, provider: 'provider_b', name: 'feed_d', event_types: [:unknown] ))
+        feed_registry.register(build(:feed, provider: 'provider_a', name: 'feed_c', event_types: [:unknown] ))
+        feed_registry.register(build(:feed, provider: 'provider_b', name: 'feed_a', event_types: [:unknown] ))
+        feed_registry.register(build(:feed, provider: 'provider_b', name: 'feed_b', event_types: [:unknown] ))
+        feed_registry.register(build(:feed, provider: 'provider_a', name: 'feed_b', event_types: [:unknown] ))
+
+        output = temp_stdout do
+          action.exec
+        end
+
+        data = MultiJson.load(output)
+        expect(data).to match(
+          [
+            {'provider' => 'provider_a', 'name' => 'feed_a', 'type' => kind_of(String), 'link' => kind_of(String)},
+            {'provider' => 'provider_a', 'name' => 'feed_b', 'type' => kind_of(String), 'link' => kind_of(String)},
+            {'provider' => 'provider_a', 'name' => 'feed_c', 'type' => kind_of(String), 'link' => kind_of(String)},
+            {'provider' => 'provider_a', 'name' => 'feed_d', 'type' => kind_of(String), 'link' => kind_of(String)},
+            {'provider' => 'provider_b', 'name' => 'feed_a', 'type' => kind_of(String), 'link' => kind_of(String)},
+            {'provider' => 'provider_b', 'name' => 'feed_b', 'type' => kind_of(String), 'link' => kind_of(String)},
+            {'provider' => 'provider_b', 'name' => 'feed_c', 'type' => kind_of(String), 'link' => kind_of(String)},
+            {'provider' => 'provider_b', 'name' => 'feed_d', 'type' => kind_of(String), 'link' => kind_of(String)}
+          ]
+        )
+      end
+    end
+  end
+
+  context "when list.format is not set" do
+    it_behaves_like 'table output'
+  end
+  context "when list.format == 'table'" do
+    before :each do
+      config.format = 'table'
+    end
+    it_behaves_like 'table output'
+  end
+
+  context "when list.format == 'json'" do
+    before :each do
+      config.format = 'json'
+    end
+    it_behaves_like 'json output'
+  end
+
+end

data/spec/threatinator/actions/run/action_spec.rb

@@ -0,0 +1,106 @@
+require 'spec_helper'
+require 'threatinator/actions/run/action'
+require 'threatinator/actions/run/config'
+require 'threatinator/plugin_loader'
+
+describe Threatinator::Actions::Run::Action do
+  let(:feed_registry) { build(:feed_registry) }
+  let(:feed_runner) { double('feed runner') }
+  let(:plugin_loader) { Threatinator::PluginLoader.new }
+  let(:config_class) { Threatinator::Actions::Run::Config.generate(plugin_loader) }
+  let(:config) { config_class.new }
+  let(:action) { described_class.new(feed_registry, config) }
+  let(:feed) { build(:feed, provider: "my_provider", name: "my_name") }
+
+  before :each do
+    feed_registry.register(feed)
+  end
+
+  context "when configured with feed provider and name that exists within the registry" do
+    let(:output) { double('mock output') }
+    let(:observer) { double('observer') }
+    before :each do
+      config.feed_provider = "my_provider"
+      config.feed_name = "my_name"
+
+      allow(feed_registry).to receive(:get).and_call_original
+      allow(config.output).to receive(:build_output).and_return(output)
+      allow(Threatinator::FeedRunner).to receive(:new).and_return(feed_runner)
+      allow(feed_runner).to receive(:run)
+      allow(feed_runner).to receive(:add_observer)
+    end
+
+    describe "#exec" do
+
+      it "queries the feed registry for the provider and name" do
+        action.exec
+        expect(feed_registry).to have_received(:get).with("my_provider", "my_name")
+      end
+
+      it "builds the output using config.output.build_output" do
+        action.exec
+        expect(config.output).to have_received(:build_output)
+      end
+
+      it "runs the feed" do
+        action.exec
+        expect(feed_runner).to have_received(:run)
+      end
+
+      context "when a record was missed" do
+        let(:status_observer) { Threatinator::Actions::Run::StatusObserver.new }
+        before :each do
+          allow(Threatinator::Actions::Run::StatusObserver).to receive(:new).and_return(status_observer)
+          allow(status_observer).to receive(:missed?).and_return(true)
+        end
+
+        it "logs an error message" do
+          expect(action.logger).to receive(:error).with(/records were MISSED/)
+          action.exec
+        end
+      end
+
+      context "when a record had an error" do
+        let(:status_observer) { Threatinator::Actions::Run::StatusObserver.new }
+        before :each do
+          allow(Threatinator::Actions::Run::StatusObserver).to receive(:new).and_return(status_observer)
+          allow(status_observer).to receive(:errors?).and_return(true)
+        end
+
+        it "logs an error message" do
+          expect(action.logger).to receive(:error).with(/records had errors/)
+          action.exec
+        end
+      end
+    end
+
+    context "when configured with an observer" do
+      before :each do
+        allow(feed_runner).to receive(:add_observer)
+        config.observers = [ observer ]
+      end
+
+      describe "#exec" do
+        it "adds the observer to FeedRunner" do
+          expect(feed_runner).to receive(:add_observer).with(observer)
+          action.exec
+        end
+      end
+    end
+  end
+
+  context "when the registry does not contain the configured feed_provider or feed_name" do
+    before :each do
+      config.feed_provider = "unknown_provider"
+      config.feed_name = "unknown_feed_name"
+    end
+    describe "#exec" do
+      it "raises Threatinator::Exceptions::UnknownFeed" do
+        expect {
+          action.exec
+        }.to raise_error(Threatinator::Exceptions::UnknownFeed) 
+      end
+    end
+  end
+end
+

data/spec/threatinator/actions/run/config_spec.rb

@@ -0,0 +1,39 @@
+require 'spec_helper'
+require 'threatinator/actions/run/config'
+require 'threatinator/plugin_loader'
+require 'fixtures/plugins/fake'
+
+describe Threatinator::Actions::Run::Config do
+
+  let(:plugin_loader) { Threatinator::PluginLoader.new }
+
+  describe ".generate(plugin_loader)" do
+    before :each do
+      allow(Threatinator::Actions::Run::OutputConfig).to receive(:generate).and_call_original
+      plugin_loader.register_plugin(:output, :plugin1, FakeOutputPlugins::Plugin1)
+      plugin_loader.register_plugin(:output, :plugin2, FakeOutputPlugins::Plugin2)
+      plugin_loader.register_plugin(:output, :plugin3, FakeOutputPlugins::Plugin3)
+      @generated_class = described_class.generate(plugin_loader)
+    end
+
+    let(:generated_class) { @generated_class }
+
+    it "returns a subclass of Threatinator::Config::Base" do
+      expect(generated_class.superclass).to be(Threatinator::Config::Base)
+    end
+
+    it "generates a new Output config class using the plugin_loader" do
+      expect(Threatinator::Actions::Run::OutputConfig).to have_received(:generate).with(plugin_loader)
+    end
+
+    describe "an instance" do
+      let(:config) { generated_class.new }
+      describe "#output" do
+        specify "returns an instance of a subclass of Threatinator::Config::Base" do
+          expect(config.output.class.superclass).to be(Threatinator::Config::Base)
+        end
+      end
+    end
+  end
+end
+

data/spec/threatinator/actions/run/coverage_observer_spec.rb

@@ -0,0 +1,151 @@
+require 'spec_helper'
+require 'threatinator/actions/run/coverage_observer'
+
+describe Threatinator::Actions::Run::CoverageObserver do
+  before :each do
+    @tmpdir = Dir.mktmpdir
+  end
+
+  after :each do
+    observer.update(:end)
+    FileUtils.remove_entry_secure @tmpdir
+  end
+
+  let(:filename) { File.join(@tmpdir, "coverage.csv") }
+  let(:observer) { described_class.new(filename) }
+
+  it_should_behave_like "a FeedRunner observer"
+
+  context "#update(:start)" do
+    it "creates the file specified by filename" do
+      expect(File.exist?(filename)).to eq(false)
+      observer.update(:start)
+      expect(File.exist?(filename)).to eq(true)
+    end
+  end
+
+  context "#update(:end)" do
+    before :each do
+      observer.update(:start)
+    end
+
+    context "when at least one record has been written" do
+      before :each do
+        observer.update(:record_parsed, build(:record), [ build(:event) ])
+        observer.update(:end)
+      end
+
+      specify "the first line is the header" do
+        data = File.read(filename)
+        expect(data.lines.to_a.first).to eq("status,event_count,line_number,pos_start,pos_end,data,message\n")
+      end
+
+      it "closes the file so that no more records will be written" do
+        data_before = File.read(filename)
+        10.times do
+          observer.update(:record_missed, build(:record))
+        end
+        data_after = File.read(filename)
+        expect(data_before).to eq(data_after)
+      end
+    end
+  end
+
+  context "#update(:record_filtered, record)" do
+    before :each do
+      observer.update(:start)
+    end
+
+    it "writes a csv entry to the file indicating that it was filtered" do
+      record = build(:record, line_number: 23, pos_start: 99, pos_end: 105, data: "foobar\r\n")
+      observer.update(:record_filtered, record)
+      observer.update(:end)
+      csv = CSV.read(filename, headers: true, header_converters: :symbol)
+      expect(csv[-1].to_hash).to eq(
+        status: "filtered",
+        event_count: "0",
+        line_number: "23",
+        pos_start: "99",
+        pos_end: "105",
+        data: '"foobar\r\n"',
+        message: ''
+      )
+    end
+  end
+
+  context "#update(:record_filtered, record)" do
+    before :each do
+      observer.update(:start)
+    end
+
+    it "writes a csv entry to the file indicating that it was missed" do
+      record = build(:record, line_number: 22, pos_start: 98, pos_end: 104, data: "blabla\r\n")
+      observer.update(:record_filtered, record)
+      observer.update(:end)
+      csv = CSV.read(filename, headers: true, header_converters: :symbol)
+      expect(csv[-1].to_hash).to eq(
+        status: "filtered",
+        event_count: "0",
+        line_number: "22",
+        pos_start: "98",
+        pos_end: "104",
+        data: '"blabla\r\n"',
+        message: ''
+      )
+    end
+  end
+
+  context "#update(:record_parsed, record, events)" do
+    before :each do
+      observer.update(:start)
+    end
+
+    let(:record) { build(:record, line_number: 1, pos_start: 0, pos_end: 10, data: "woofwoof\r\n") }
+    let(:events) { [ build(:event), build(:event) ] }
+
+    it "writes a csv entry to the file indicating that it was parsed, with the number of events" do
+      observer.update(:record_parsed, record, events)
+      observer.update(:end)
+      csv = CSV.read(filename, headers: true, header_converters: :symbol)
+      expect(csv[-1].to_hash).to eq(
+        status: "parsed",
+        event_count: "2",
+        line_number: "1",
+        pos_start: "0",
+        pos_end: "10",
+        data: '"woofwoof\r\n"',
+        message: ''
+      )
+    end
+  end
+
+  context "#update(:record_error, record, errors)" do
+    before :each do
+      observer.update(:start)
+    end
+
+    let(:record) { build(:record, line_number: 1, pos_start: 0, pos_end: 10, data: "woofwoof\r\n") }
+    let(:errors) { 
+      [
+        Threatinator::Exceptions::EventBuildError.new("error 1"),
+        Threatinator::Exceptions::EventBuildError.new("error 2"),
+        Threatinator::Exceptions::EventBuildError.new("error 3")
+      ]
+    }
+
+    it "writes a csv entry to the file indicating it encountered an error, with the error messages" do
+      observer.update(:record_error, record, errors)
+      observer.update(:end)
+      csv = CSV.read(filename, headers: true, header_converters: :symbol)
+      expect(csv[-1].to_hash).to eq(
+        status: "error",
+        event_count: "0",
+        line_number: "1",
+        pos_start: "0",
+        pos_end: "10",
+        data: '"woofwoof\r\n"',
+        message: 'error 1, error 2, error 3'
+      )
+    end
+  end
+end

data/spec/threatinator/actions/run/output_config_spec.rb

@@ -0,0 +1,89 @@
+require 'spec_helper'
+require 'threatinator/actions/run/output_config'
+require 'threatinator/plugin_loader'
+require 'fixtures/plugins/fake'
+
+describe Threatinator::Actions::Run::OutputConfig do
+  let(:plugin_loader) { Threatinator::PluginLoader.new }
+
+  describe ".generate(plugin_loader)" do
+    before :each do
+      plugin_loader.register_plugin(:output, :plugin1, FakeOutputPlugins::Plugin1)
+      plugin_loader.register_plugin(:output, :plugin2, FakeOutputPlugins::Plugin2)
+      plugin_loader.register_plugin(:output, :plugin3, FakeOutputPlugins::Plugin3)
+    end
+
+    let(:generated_class) { described_class.generate(plugin_loader) }
+    it "returns a subclass of Threatinator::Config::Base" do
+      expect(generated_class.superclass).to be(Threatinator::Config::Base)
+    end
+
+    describe "#formats" do
+      it "returns an array containing all the registered output format names" do
+        expect(generated_class.formats).to contain_exactly('plugin1', 'plugin2', 'plugin3')
+      end
+    end
+
+    describe "#formats_str" do
+      it "returns a string of the #formats sorted alphabetically" do
+        expect(generated_class.formats_str).to eq('plugin1, plugin2, plugin3')
+      end
+    end
+
+    describe "attributes" do
+      describe ":format" do
+        describe "description" do
+          it "should describe the output and the available formats" do
+            a = generated_class.attribute_set[:format]
+            desc_proc = a.options[:description]
+            ret = desc_proc.call(generated_class, a)
+            expect(ret).to eq('Output format (plugin1, plugin2, plugin3)')
+          end
+        end
+      end
+    end
+
+    describe "an instance" do
+      let(:config) { generated_class.new }
+      specify "has an attribute for each plugin name that is an instance of that plugin's config" do
+        expect(config.plugin1).to be_a(FakeOutputPlugins::Plugin1::Config)
+        expect(config.plugin1).to respond_to(:foo)
+        expect(config.plugin2).to be_a(FakeOutputPlugins::Plugin2::Config)
+        expect(config.plugin2).to respond_to(:bar)
+        expect(config.plugin3).to be_a(FakeOutputPlugins::Plugin3::Config)
+        expect(config.plugin3).to respond_to(:woof)
+      end
+
+      describe "#build_output" do
+        it "raises UnknownPlugin when no plugin is found for the value of #format" do
+          config.format = :asdf
+          expect {
+            config.build_output
+          }.to raise_error(Threatinator::Exceptions::UnknownPlugin)
+        end
+
+        it "raises CouldNotFindOutputConfigError if there is no config for the #format" do
+          config.format = :plugin1
+          config.plugin1 = nil
+          expect {
+            config.build_output
+          }.to raise_error(Threatinator::Exceptions::CouldNotFindOutputConfigError)
+        end
+
+        context "when #format is associated with a reigstered plugin" do
+          it "initializes the output plugin with the plugin's config" do
+            config.format = :plugin2
+            expect(FakeOutputPlugins::Plugin2).to receive(:new).with(config.plugin2)
+            config.build_output
+          end
+
+          it "returns an instance of the output plugin" do
+            config.format = :plugin3
+            expect(config.build_output).to be_a(FakeOutputPlugins::Plugin3)
+          end
+        end
+      end
+    end
+  end
+end
+