threatinator 0.1.1

data/spec/threatinator/plugins/output/csv_spec.rb

@@ -0,0 +1,46 @@
+require 'spec_helper'
+require 'threatinator/plugins/output/csv'
+require 'stringio'
+
+describe Threatinator::Plugins::Output::Csv do
+  let(:config) { Threatinator::Plugins::Output::Csv::Config.new }
+  it_should_behave_like "a file-based output plugin", :csv
+
+  describe "the output" do
+    let(:io) { StringIO.new }
+    let(:output) { described_class.new(config) }
+
+    before :each do
+      config.io = io
+      events.each do |e|
+        output.handle_event(e)
+      end
+    end
+
+    let(:lines) { io.string.lines.to_a }
+    let(:events) { 
+      ret = []
+      1.upto(10) do |i|
+        ret << build(:event, feed_provider: "my_prov#{i}", 
+              feed_name: "my_name#{i}", type: :scanning,
+              ipv4s: ["#{i}.1.1.1","#{i}.1.1.2","#{i}.1.1.3","#{i}.1.1.4","#{i}.1.1.5"],
+              fqdns: ["a#{i}.com","b#{i}.com","c#{i}.com","d#{i}.com","e#{i}.com"]
+             ) 
+      end
+      ret
+    }
+
+
+    describe "the header" do
+      subject { lines.first }
+      it { is_expected.to eq("provider,feed_name,type,ipv4_1,ipv4_2,ipv4_3,ipv4_4,fqdn_1,fqdn_2,fqdn_3,fqdn_4\n") }
+    end
+
+    it "outputs each event, limited to the first four IPs and four fqdns" do
+      line2 = lines[1]
+      1.upto(10) do |i|
+        expect(lines[i]).to eq("my_prov#{i},my_name#{i},scanning,#{i}.1.1.1,#{i}.1.1.2,#{i}.1.1.3,#{i}.1.1.4,a#{i}.com,b#{i}.com,c#{i}.com,d#{i}.com\n")
+      end
+    end
+  end
+end

data/spec/threatinator/plugins/output/null_spec.rb

@@ -0,0 +1,17 @@
+require 'spec_helper'
+require 'threatinator/plugins/output/null'
+require 'stringio'
+
+describe Threatinator::Plugins::Output::Null do
+  let(:config) { Threatinator::Plugins::Output::Null::Config.new }
+  it_should_behave_like "an output plugin", :null
+
+  describe "#handle_event" do
+    let(:output) { described_class.new(config) }
+    it "does not call any methods on the event" do
+      event = double("event")
+      output.handle_event(event)
+    end
+  end
+end
+

data/spec/threatinator/plugins/output/rubydebug_spec.rb

@@ -0,0 +1,37 @@
+require 'spec_helper'
+require 'threatinator/plugins/output/rubydebug'
+require 'stringio'
+
+describe Threatinator::Plugins::Output::Rubydebug do
+  let(:config) { Threatinator::Plugins::Output::Rubydebug::Config.new }
+
+  it_should_behave_like "a file-based output plugin", :rubydebug
+
+  describe "the output" do
+    let(:output) { described_class.new(config) }
+    let(:io) { StringIO.new }
+    before :each do
+      config.io = io
+      events.each do |e|
+        output.handle_event(e)
+      end
+    end
+
+    let(:events) { 
+      ret = []
+      1.upto(10) do |i|
+        ret << build(:event, feed_provider: "my_prov#{i}", 
+              feed_name: "my_name#{i}", type: :scanning,
+              ipv4s: ["#{i}.1.1.1","#{i}.1.1.2","#{i}.1.1.3","#{i}.1.1.4","#{i}.1.1.5"],
+              fqdns: ["a#{i}.com","b#{i}.com","c#{i}.com","d#{i}.com","e#{i}.com"]
+             ) 
+      end
+      ret
+    }
+
+    it "outputs a bunch of stuff" do
+      expect(io.string.length).to be > 0
+    end
+  end
+end
+

data/spec/threatinator/property_definer_spec.rb

@@ -0,0 +1,155 @@
+require 'spec_helper'
+require 'threatinator/property_definer'
+
+describe Threatinator::PropertyDefiner do
+  describe "defining a property" do
+    let(:klass) {
+      Class.new do
+        include Threatinator::PropertyDefiner
+        property :my_prop
+      end
+    }
+    it "should add a setter for the property you define" do
+      klass = Class.new do
+        include Threatinator::PropertyDefiner
+        property :some_prop
+      end
+      expect(klass.new).to respond_to(:some_prop=)
+    end
+    it "should add a getter for the property you define" do
+      klass = Class.new do
+        include Threatinator::PropertyDefiner
+        property :another_prop
+      end
+      expect(klass.new).to respond_to(:another_prop)
+    end
+  end
+
+  describe "getting and setting properties" do
+    let(:klass) {
+      Class.new do
+        include Threatinator::PropertyDefiner
+        property :my_prop
+      end
+    }
+    it "should let me set the value of a property and retrieve it" do
+      instance = klass.new
+      instance.my_prop = 1234
+      expect(instance.my_prop).to eq(1234)
+    end
+    it "the getter should respond with nil by default" do
+      instance = klass.new
+      expect(instance.my_prop).to be_nil
+    end
+  end
+  describe ":type (type validation)" do
+    context "when :type is set to a class" do
+      let(:klass) {
+        Class.new do
+          include Threatinator::PropertyDefiner
+          property :my_prop, type: Integer
+        end
+      }
+      let(:instance) { klass.new }
+
+      it "should not raise an error if the property is set to a instance of the specified class" do
+        expect{instance.my_prop = 1234}.not_to raise_error
+      end
+
+      it "should raise an InvalidAttributeError if the property being set is not of its defined type" do
+        expect{instance.my_prop = "asdf"}.to raise_error do |e|
+          expect(e).to be_a(Threatinator::Exceptions::InvalidAttributeError)
+          expect(e.attribute).to eq(:my_prop)
+        end
+      end
+    end
+  end
+
+  describe ":validate" do
+    it "should raise an ArgumentError if :validate is not a proc" do
+      expect {
+        Class.new do
+          include Threatinator::PropertyDefiner
+          property :my_prop, validate: 1234
+        end
+      }.to raise_error(ArgumentError)
+
+    end
+
+    it "should call the validate block whenever the associated attribute is set" do
+      expect { |b|
+        klass = Class.new do
+          include Threatinator::PropertyDefiner
+          property :my_prop, validate: lambda { |obj, val| b.to_proc.call(obj, val); true }
+        end
+        instance = klass.new
+        instance.my_prop = "value1"
+        instance.my_prop = "value2"
+      }.to yield_successive_args(
+        [kind_of(Threatinator::PropertyDefiner), "value1"], 
+        [kind_of(Threatinator::PropertyDefiner), "value2"])
+    end
+
+    it "should raise an InvalidAttributeError if the validation block returns false" do
+      klass = Class.new do
+        include Threatinator::PropertyDefiner
+        property :my_prop, validate: lambda { |obj, val| false }
+      end
+      instance = klass.new
+      expect { |b|
+        instance.my_prop = "value1"
+      }.to raise_error do |e|
+        expect(e).to be_a(Threatinator::Exceptions::InvalidAttributeError)
+        expect(e.attribute).to eq(:my_prop)
+      end
+    end
+    it "should not raise an error if the validation block returns true" do
+      klass = Class.new do
+        include Threatinator::PropertyDefiner
+        property :my_prop, validate: lambda { |obj, val| true }
+      end
+      instance = klass.new
+      expect { |b|
+        instance.my_prop = "value1"
+      }.not_to raise_error
+    end
+  end
+
+  describe ":default" do
+    context "when :default is set to a value" do
+      it "should return the object when accessed via a getter" do
+        expected_object = "my_val"
+        klass = Class.new do
+          include Threatinator::PropertyDefiner
+          property :my_prop, default: expected_object
+        end
+        instance = klass.new
+        expect(instance.my_prop).to be(expected_object)
+      end
+    end
+    context "when :default is set to a proc" do
+      it "should not call the proc if a value is provided" do
+        expect { |b| 
+          klass = Class.new do
+            include Threatinator::PropertyDefiner
+            property :my_prop, default: b.to_proc
+          end
+          instance = klass.new
+          instance.my_prop = 1234
+        }.not_to yield_control
+      end
+      it "should only call the proc once, and only when it is accessed" do
+        expect { |b| 
+          klass = Class.new do
+            include Threatinator::PropertyDefiner
+            property :my_prop, default: lambda { b.to_proc.call(); }
+          end
+          instance = klass.new
+          val1 = instance.my_prop
+          val2 = instance.my_prop
+          val3 = instance.my_prop
+        }.to yield_control.exactly(1).times
+      end
+    end
+  end
+end

data/spec/threatinator/record_spec.rb

@@ -0,0 +1,19 @@
+require 'spec_helper'
+require 'threatinator/record'
+
+describe Threatinator::Record do
+  it_should_behave_like 'a record' do
+    let(:data) { "asdf" }
+    let(:opts) { { } }
+  end
+  context "two instances with different data" do
+    it_should_behave_like 'a record when compared to a differently configured record' do
+      let(:data) { "foo" }
+      let(:opts) { { } }
+
+      let(:data2) { "bar" }
+      let(:opts2) { {} }
+    end
+  end
+
+end

data/spec/threatinator/registry_spec.rb

@@ -0,0 +1,97 @@
+require 'spec_helper'
+require 'threatinator/registry'
+
+describe Threatinator::Registry do
+  let(:registry) { described_class.new }
+  let(:ten_things) { {
+    a: Object.new,
+    b: Object.new,
+    c: Object.new,
+    d: Object.new,
+    e: Object.new,
+    f: Object.new,
+    g: Object.new,
+    h: Object.new,
+    i: Object.new,
+    j: Object.new
+  } }
+
+  describe "#clear" do
+    it "should remove all existing registrations" do
+      expect(registry.count).to eq(0)
+      registry.register(:foo, 123)
+      registry.register(:bar, 456)
+      expect(registry.count).to eq(2)
+      registry.clear
+      expect(registry.count).to eq(0)
+    end
+  end
+
+  describe "#keys" do
+    it "returns an array containing all of the keys" do
+      expect(registry.keys).to eq([])
+      registry.register(:foo, Object.new)
+      registry.register(:bar, Object.new)
+      registry.register(:woof, Object.new)
+      expect(registry.keys).to contain_exactly(:foo, :bar, :woof)
+    end
+  end
+
+  describe "#register" do
+    it "should register the provided object to the given key" do
+      obj = Object.new
+      registry.register(:foo, obj)
+      expect(registry.get(:foo)).to be(obj)
+    end
+
+    it "should return the object that was registered" do
+      obj = Object.new
+      expect(registry.register(:foo, obj)).to be(obj)
+    end
+
+    it "should raise a AlreadyRegisteredError if something is already registered with the the given key" do
+      obj = Object.new
+      registry.register(:foo, obj)
+      expect { 
+        registry.register(:foo, obj)
+      }.to raise_error(Threatinator::Exceptions::AlreadyRegisteredError)
+    end
+  end
+
+  describe "#each" do
+    it "should enumerate through each key and registered object" do
+      ten_things.each_pair do |i, thing|
+        registry.register(i, thing)
+      end
+      found_objects = []
+      registry.each do |key, obj|
+        found_objects << [ key, obj ]
+      end
+      expect(found_objects).to match_array(ten_things.to_a)
+    end
+  end
+
+  describe "#count" do
+    it "should return the number of objects contained within the registry" do
+      expect(registry.count).to eq(0)
+      ten_things.each_pair do |i, thing|
+        registry.register(i, thing)
+      end
+      expect(registry.count).to eq(10)
+    end
+  end
+
+  describe "#get" do
+    it "should return nil if key isn't registered" do
+      expect(registry.get(:foo)).to be_nil
+    end
+
+    it "should return the correct feed for the key" do
+      ten_things.each_pair { |i, o| registry.register(i, o) }
+      obj = Object.new
+      registry.register(:foo, obj)
+      expect(registry.get(:foo)).to be(obj)
+    end
+  end
+end
+

data/spec/threatinator/runner_spec.rb

@@ -0,0 +1,273 @@
+#require 'spec_helper'
+#require 'threatinator/runner'
+#
+#describe Threatinator::Runner do
+#  let(:runner) { Threatinator::Runner.new }
+#
+#  def generate_feedfile(filename, provider, name, url = "https://foobar/#{provider}/#{name}.data")
+#    File.open(filename, "w") do |fio|
+#      fio.write <<EOS
+#provider "#{provider}"
+#name "#{name}"
+#fetch_http('#{url}')
+#
+#parse_eachline(:separator => "\n") do |builder, line|
+#end
+#EOS
+#    end
+#  end
+#
+#  describe "#add_feed_path" do
+#    it "add the paths to #feed_paths" do
+#      expect(runner.feed_paths).to eq([])
+#      runner.add_feed_path("/foo/bar1")
+#      expect(runner.feed_paths).to eq(["/foo/bar1"])
+#      runner.add_feed_path("/foo/bar2")
+#      expect(runner.feed_paths).to eq(["/foo/bar1", "/foo/bar2"])
+#    end
+#  end
+#
+#  describe "#_load_feeds" do
+#    before :each do
+#      @feed_path1 = Dir.mktmpdir
+#      @feed_path2 = Dir.mktmpdir
+#    end
+#
+#    after :each do
+#      FileUtils.remove_entry_secure @feed_path1
+#      FileUtils.remove_entry_secure @feed_path2
+#    end
+#
+#    context "with no feed paths" do
+#      it "should not have loaded any feeds" do
+#        runner._load_feeds
+#        expect(runner.registry.count).to eq(0)
+#      end
+#    end
+#
+#    context "with paths added to the runner" do
+#      before :each do
+#        runner.add_feed_path(@feed_path1)
+#        runner.add_feed_path(@feed_path2)
+#      end
+#
+#      it "should load feeds from all of the configured paths" do
+#        5.times do |i|
+#          generate_feedfile(File.join(@feed_path1, "feed#{i}.feed"), "provider1", "feed#{i}")
+#          generate_feedfile(File.join(@feed_path2, "feed#{i}.feed"), "provider2", "feed#{i}")
+#        end
+#        runner._load_feeds
+#        expect(runner.registry.count).to eq(10)
+#      end
+#
+#      it "should ignore files that don't end with .feed" do
+#        generate_feedfile(File.join(@feed_path1, "feed1.fee"), "provider1", "feed1")
+#        generate_feedfile(File.join(@feed_path1, "feed1.fed"), "provider1", "feed2")
+#        generate_feedfile(File.join(@feed_path1, "feed1.rb"), "provider1", "feed3")
+#        generate_feedfile(File.join(@feed_path1, "feed1.feed"), "real_provider", "my_feed")
+#        runner._load_feeds
+#        expect(runner.registry.count).to eq(1)
+#        expect(runner.registry.get("real_provider", "my_feed")).to be_a(Threatinator::Feed)
+#        expect(runner.registry.get("provider1", "feed1")).to be_nil
+#        expect(runner.registry.get("provider1", "feed2")).to be_nil
+#        expect(runner.registry.get("provider1", "feed3")).to be_nil
+#      end
+#
+#      it "should recurse subdirectories, loading feeds from there" do
+#        level1 = File.join(@feed_path1, "level1")
+#        level2 = File.join(@feed_path1, "level1", "level2")
+#        level3 = File.join(@feed_path1, "level1", "level2", "level3")
+#        FileUtils.mkdir_p level1
+#        FileUtils.mkdir_p level2
+#        FileUtils.mkdir_p level3
+#        generate_feedfile(File.join(level1, "feed1.feed"), "provider1", "feed1")
+#        generate_feedfile(File.join(level2, "feed2.feed"), "provider1", "feed2")
+#        generate_feedfile(File.join(level3, "feed3.feed"), "provider1", "feed3")
+#        runner._load_feeds
+#        expect(runner.registry.count).to eq(3)
+#      end
+#
+#      it "should raise an exception if the same feed provider/name combination appears in multiple files" do
+#        generate_feedfile(File.join(@feed_path1, "feed1.feed"), "provider1", "feed1")
+#        generate_feedfile(File.join(@feed_path1, "feed2.feed"), "provider1", "feed1")
+#        expect {
+#          runner._load_feeds
+#        }.to raise_error(Threatinator::Exceptions::AlreadyRegisteredError)
+#      end
+#
+#    end
+#  end
+#
+#  describe "#_register_feed_from_file" do
+#    let(:feedfile) {FEED_FIXTURES.join("provider1", "feed1.feed").to_s}
+#
+#    it "should return the feed after parsing the file" do
+#      ret = runner._register_feed_from_file(feedfile)
+#      expect(ret).to be_a(Threatinator::Feed)
+#      expect(ret.provider).to eq("provider1")
+#      expect(ret.name).to eq("feed1")
+#    end
+#
+#    it "should have registered the feed" do
+#      expect(runner.registry.count).to eq(0)
+#      feed = runner._register_feed_from_file(feedfile)
+#      expect(runner.registry.count).to eq(1)
+#      expect(runner.registry.get(feed.provider, feed.name)).to be(feed)
+#    end
+#  end
+#
+#  describe "#list" do
+#    let(:io_out) { StringIO.new }
+#
+#    before :each do
+#      @feed_path = Dir.mktmpdir
+#      runner.add_feed_path(@feed_path)
+#    end
+#
+#    after :each do
+#      FileUtils.remove_entry_secure @feed_path
+#    end
+#
+#    context "with no feed paths" do
+#      it "should output the header" do
+#        runner.list(io_out: io_out);
+#        lines = io_out.string.lines.to_a
+#        expect(lines[0]).to eq("provider  name  type  link/path\n")
+#        expect(lines[1]).to eq("--------  ----  ----  ---------\n")
+#      end
+#
+#      it "should output the footer with a total of 0" do
+#        runner.list(io_out: io_out);
+#        lines = io_out.string.lines.to_a
+#        expect(lines[-2]).to eq("--------  ----  ----  ---------\n")
+#        expect(lines[-1]).to eq("Total: 0\n")
+#      end
+#    end
+#
+#    context "with paths added to the runner" do
+#      before :each do
+#      end
+#      describe "the header row, header separator, and footer separator" do
+#        it "should vary the width of 'provider' based on the longest provider name" do
+#          generate_feedfile(File.join(@feed_path, "feed.feed"), "A" * 10, "x", "http://x")
+#          generate_feedfile(File.join(@feed_path, "feed.feed"), "A" * 20, "x", "http://x")
+#          generate_feedfile(File.join(@feed_path, "feed.feed"), "A" * 30, "x", "http://x")
+#
+#          runner.list(io_out: io_out);
+#          lines = io_out.string.lines.to_a
+#          expect(lines[0]).to eq("provider                        name  type  link/path\n")
+#          expect(lines[1]).to eq("------------------------------  ----  ----  ---------\n")
+#          expect(lines[-2]).to eq("------------------------------  ----  ----  ---------\n")
+#        end
+#
+#        it "should vary the width of 'name' based on the longest feed name" do
+#          generate_feedfile(File.join(@feed_path, "feed.feed"), 'a', "A" * 10, "http://x")
+#          generate_feedfile(File.join(@feed_path, "feed.feed"), 'a', "A" * 20, "http://x")
+#          generate_feedfile(File.join(@feed_path, "feed.feed"), 'a', "A" * 30, "http://x")
+#
+#          runner.list(io_out: io_out);
+#          lines = io_out.string.lines.to_a
+#          expect(lines[0]).to eq("provider  name                            type  link/path\n")
+#          expect(lines[1]).to eq("--------  ------------------------------  ----  ---------\n")
+#          expect(lines[-2]).to eq("--------  ------------------------------  ----  ---------\n")
+#        end
+#
+#        it "should vary the width of 'link/path' based on the longest link name" do
+#          generate_feedfile(File.join(@feed_path, "feed.feed"), 'a', 'b', "http://" + ("A" * 10))
+#          generate_feedfile(File.join(@feed_path, "feed.feed"), 'a', 'b', "http://" + ("A" * 20))
+#          generate_feedfile(File.join(@feed_path, "feed.feed"), 'a', 'b', "http://" + ("A" * 30))
+#
+#          runner.list(io_out: io_out);
+#          lines = io_out.string.lines.to_a
+#          expect(lines[0]).to eq("provider  name  type  link/path                            \n")
+#          expect(lines[1]).to eq("--------  ----  ----  -------------------------------------\n")
+#          expect(lines[-2]).to eq("--------  ----  ----  -------------------------------------\n")
+#        end
+#      end
+#
+#      describe "the list of feeds" do
+#        it "should be sorted by provider name and then feed name" do
+#          generate_feedfile(File.join(@feed_path, "feed1.feed"), 'provider_b', 'feed_c')
+#          generate_feedfile(File.join(@feed_path, "feed2.feed"), 'provider_a', 'feed_d')
+#          generate_feedfile(File.join(@feed_path, "feed3.feed"), 'provider_a', 'feed_a')
+#          generate_feedfile(File.join(@feed_path, "feed4.feed"), 'provider_b', 'feed_d')
+#          generate_feedfile(File.join(@feed_path, "feed5.feed"), 'provider_a', 'feed_c')
+#          generate_feedfile(File.join(@feed_path, "feed6.feed"), 'provider_b', 'feed_a')
+#          generate_feedfile(File.join(@feed_path, "feed7.feed"), 'provider_b', 'feed_b')
+#          generate_feedfile(File.join(@feed_path, "feed8.feed"), 'provider_a', 'feed_b')
+#
+#          runner.list(io_out: io_out);
+#          lines = io_out.string.lines.to_a
+#          expect(lines[2]).to eq("provider_a  feed_a  http  https://foobar/provider_a/feed_a.data\n")
+#          expect(lines[3]).to eq("provider_a  feed_b  http  https://foobar/provider_a/feed_b.data\n")
+#          expect(lines[4]).to eq("provider_a  feed_c  http  https://foobar/provider_a/feed_c.data\n")
+#          expect(lines[5]).to eq("provider_a  feed_d  http  https://foobar/provider_a/feed_d.data\n")
+#
+#          expect(lines[6]).to eq("provider_b  feed_a  http  https://foobar/provider_b/feed_a.data\n")
+#          expect(lines[7]).to eq("provider_b  feed_b  http  https://foobar/provider_b/feed_b.data\n")
+#          expect(lines[8]).to eq("provider_b  feed_c  http  https://foobar/provider_b/feed_c.data\n")
+#          expect(lines[9]).to eq("provider_b  feed_d  http  https://foobar/provider_b/feed_d.data\n")
+#        end
+#      end
+#
+#      describe "the footer" do
+#        it "should indicate the number of feeds" do
+#          20.times do |i|
+#            generate_feedfile(File.join(@feed_path, "feed#{i}.feed"), "a#{i}", 'b', "http://" + ("A" * 10))
+#          end
+#          runner.list(io_out: io_out);
+#          lines = io_out.string.lines.to_a
+#          expect(lines[-1]).to eq("Total: 20\n")
+#        end
+#      end
+#    end
+#  end
+#
+#  describe "#run" do
+#    before :each do
+#      @feed_path = Dir.mktmpdir
+#      generate_feedfile(File.join(@feed_path, "feed1.feed"), "provider1", "feed1")
+#      runner.add_feed_path(@feed_path)
+#      allow(Threatinator::FeedRunner).to receive(:run)
+#    end
+#
+#    after :each do
+#      FileUtils.remove_entry_secure @feed_path
+#    end
+#
+#    let(:loader) { 
+#      x = Threatinator::PluginLoader.new
+#      x.load_plugins(:output)
+#      x }
+#
+#
+#    let(:output) { loader[:output][:null] } 
+#
+#    it "parses all the feeds" do
+#      expect(runner).to receive(:_load_feeds).and_call_original
+#      runner.run("provider1", "feed1", output)
+#    end
+#
+#    context "when called with a provider and feed name that does not match a feed" do
+#      it "raises Threatinator::Exceptions::UnknownFeed" do
+#        expect {
+#          runner.run("foobar", "bla", output)
+#        }.to raise_error(Threatinator::Exceptions::UnknownFeed)
+#      end
+#    end
+#
+#    context "when called with a provider and feed name that matches a feed" do
+#      it "loads the feed by the given provider and name" do
+#        expect(runner.registry).to receive(:get).with("provider1", "feed1").and_call_original
+#        runner.run("provider1", "feed1", output)
+#      end
+#
+#      it "runs the feed" do
+#        opts_hash = {foo: 123}
+#        expect(Threatinator::FeedRunner).to receive(:run).with(kind_of(Threatinator::Feed), output, opts_hash)
+#        runner.run("provider1", "feed1", output, opts_hash)
+#      end
+#    end
+#  end
+#
+#end