threatinator 0.1.1

data/spec/support/helpers/io.rb

@@ -0,0 +1,11 @@
+require 'stringio'
+module IOHelpers
+  def temp_stdout
+    $stdout = StringIO.new
+    yield $stdout.string
+    return $stdout.string
+  ensure
+    $stdout = STDOUT
+  end
+end
+

data/spec/support/helpers/models.rb

@@ -0,0 +1,13 @@
+require 'threatinator/parser'
+require 'threatinator/fetcher'
+
+module FeedSpec
+  class Fetcher < Threatinator::Fetcher
+    def initialize(opts = {})
+      @io = opts[:io]
+    end
+    def fetch; @io; end
+  end
+  class Parser < Threatinator::Parser
+  end
+end

data/spec/support/shared/action_builder.rb

@@ -0,0 +1,47 @@
+require 'spec_helper'
+
+shared_examples_for "an action builder" do
+  # expects :builder
+  # expects :config_hash
+  describe "a call to #feed_registry" do
+    let(:feed_registry) { double('feed_registry') }
+    let(:feed_search_hash) { double('feed search hash') }
+    let(:feed_search) { double('feed_search') }
+
+    before :each do
+      allow(Threatinator::FeedRegistry).to receive(:build).and_return(feed_registry)
+      allow(Threatinator::Config::FeedSearch).to receive(:new).and_return(feed_search)
+    end
+
+
+    context "when config_hash['feed_search'] exists" do
+      before :each do
+        config_hash['feed_search'] = feed_search_hash
+      end
+
+      it "builds a new Threatinator::Config::FeedSearch using config_hash['feed_search']" do
+        expect(Threatinator::Config::FeedSearch).to receive(:new).with(feed_search_hash)
+        builder.feed_registry
+      end
+    end
+    context "when config_hash['feed_search'] does not exist" do
+      before :each do
+        config_hash.delete('feed_search')
+      end
+
+      it "builds a new Threatinator::Config::FeedSearch using an empty hash" do
+        expect(Threatinator::Config::FeedSearch).to receive(:new).with({})
+        builder.feed_registry
+      end
+    end
+
+    it "builds a new feed registry using the config" do
+      expect(Threatinator::FeedRegistry).to receive(:build).with(feed_search)
+      builder.feed_registry
+    end
+
+    it "returns the instance of Threatinator::FeedRegistry" do
+      expect(builder.feed_registry).to be(feed_registry)
+    end
+  end
+end

data/spec/support/shared/decoder.rb

@@ -0,0 +1,70 @@
+# encoding: utf-8
+require 'spec_helper'
+require 'threatinator/exceptions'
+
+shared_examples_for "a decoder" do
+  # Expects :encode_data_proc, :decoder_opts
+  let(:extra_opts) { { } }
+  let(:decoder) { described_class.new(decoder_opts.merge(extra_opts)) }
+
+  describe "an instance" do
+    subject { decoder }
+    it { is_expected.to respond_to(:decode) }
+
+    it "should close the IO that it decodes from" do
+      data = encode_data_proc.call("here's some data")
+      io = StringIO.new(data)
+      expect(io).not_to be_closed
+      decoder.decode(io)
+      expect(io).to be_closed
+    end
+
+  end
+
+  describe "decoding a UTF-8 string" do 
+    let(:original_string) { 
+      "\xE1\x9A\xA0\xE1\x9B\x87\xE1\x9A\xBB\xE1\x9B\xAB\xE1\x9B\x92\xE1\x9B\xA6\xE1\x9A\xA6\xE1\x9B\xAB\xE1\x9A\xA0\xE1\x9A\xB1\xE1\x9A\xA9\xE1\x9A\xA0\xE1\x9A\xA2\xE1\x9A\xB1\xE1\x9B\xAB\xE1\x9A\xA0\xE1\x9B\x81\xE1\x9A\xB1\xE1\x9A\xAA\xE1\x9B\xAB\xE1\x9A\xB7\xE1\x9B\x96\xE1\x9A\xBB\xE1\x9A\xB9\xE1\x9B\xA6\xE1\x9B\x9A\xE1\x9A\xB3\xE1\x9A\xA2\xE1\x9B\x97\n\xE1\x9B\x8B\xE1\x9A\xB3\xE1\x9B\x96\xE1\x9A\xAA\xE1\x9B\x9A\xE1\x9B\xAB\xE1\x9A\xA6\xE1\x9B\x96\xE1\x9A\xAA\xE1\x9A\xBB\xE1\x9B\xAB\xE1\x9B\x97\xE1\x9A\xAA\xE1\x9A\xBE\xE1\x9A\xBE\xE1\x9A\xAA\xE1\x9B\xAB\xE1\x9A\xB7\xE1\x9B\x96\xE1\x9A\xBB\xE1\x9A\xB9\xE1\x9B\xA6\xE1\x9B\x9A\xE1\x9A\xB3\xE1\x9B\xAB\xE1\x9B\x97\xE1\x9B\x81\xE1\x9A\xB3\xE1\x9B\x9A\xE1\x9A\xA2\xE1\x9A\xBE\xE1\x9B\xAB\xE1\x9A\xBB\xE1\x9B\xA6\xE1\x9B\x8F\xE1\x9B\xAB\xE1\x9B\x9E\xE1\x9A\xAB\xE1\x9B\x9A\xE1\x9A\xAA\xE1\x9A\xBE\n\xE1\x9A\xB7\xE1\x9B\x81\xE1\x9A\xA0\xE1\x9B\xAB\xE1\x9A\xBB\xE1\x9B\x96\xE1\x9B\xAB\xE1\x9A\xB9\xE1\x9B\x81\xE1\x9B\x9A\xE1\x9B\x96\xE1\x9B\xAB\xE1\x9A\xA0\xE1\x9A\xA9\xE1\x9A\xB1\xE1\x9B\xAB\xE1\x9B\x9E\xE1\x9A\xB1\xE1\x9B\x81\xE1\x9A\xBB\xE1\x9B\x8F\xE1\x9A\xBE\xE1\x9B\x96\xE1\x9B\xAB\xE1\x9B\x9E\xE1\x9A\xA9\xE1\x9B\x97\xE1\x9B\x96\xE1\x9B\x8B\xE1\x9B\xAB\xE1\x9A\xBB\xE1\x9B\x9A\xE1\x9B\x87\xE1\x9B\x8F\xE1\x9A\xAA\xE1\x9A\xBE\xE1\x9B\xAC\n"
+      .force_encoding("UTF-8")
+    }
+
+    let(:encoded_string) { encode_data_proc.call(original_string) }
+    let(:encoded_io) { StringIO.new(encoded_string) }
+
+    describe "the decoded data" do
+      it "should equal the original string" do
+        expect(decoder.decode(encoded_io).read()).to eq(original_string)
+      end
+      it "should be UTF-8 encoded" do
+        expect(decoder.decode(encoded_io).read().encoding).to eq(Encoding::UTF_8)
+      end
+    end
+  end
+
+  describe "decoding a UTF-8 string2" do 
+    let(:original_string) { 
+      "21826        |  Corporación Telemic C.A.,VE    |    200.75.105.49  |  2014-07-18 19:54:54  |  sshpwauth".force_encoding("UTF-8")
+    }
+
+    let(:encoded_string) { encode_data_proc.call(original_string) }
+    let(:encoded_io) { StringIO.new(encoded_string) }
+
+    describe "the decoded data" do
+      it "should equal the original string" do
+        expect(decoder.decode(encoded_io).read()).to eq(original_string)
+      end
+      it "should be UTF-8 encoded" do
+        expect(decoder.decode(encoded_io).read().encoding).to eq(Encoding::UTF_8)
+      end
+    end
+  end
+end
+
+shared_examples_for "a decoder encountering an error during decoding" do
+  # Expects :decoder, :input_io
+  it "#decode should raise a DecoderError" do
+    expect { 
+      decoder.decode(input_io)
+    }.to raise_error(Threatinator::Exceptions::DecoderError)
+  end
+end
+

data/spec/support/shared/feeds.rb

@@ -0,0 +1,218 @@
+require 'spec_helper'
+require 'threatinator/feed_builder'
+require 'threatinator/feed_runner'
+require 'threatinator/plugins/output/null'
+require 'pathname'
+
+shared_context 'a parsed record' do
+  # Expects :observer
+  before :each do
+    @record, @status, @events = observer.first
+  end
+  let(:record) { @record }
+  let(:events) { @events }
+  let(:status) { @status }
+end
+
+shared_context 'a parsed feed' do 
+  # expects :observer
+  let(:events) { observer.map { |status, record, events| events } }
+  let(:records) { observer.map { |status, record, events| record } }
+  let(:num_records) { observer.count }
+  let(:num_records_filtered) { observer.num_records_filtered }
+  let(:num_records_parsed) { observer.num_records_parsed }
+  let(:num_records_missed) { observer.num_records_missed }
+end
+
+shared_context 'for feeds', :feed => lambda { true } do
+  _feed_path = Pathname.new(self.description)
+  if _feed_path.relative?
+    # It's relative to the root of our project.
+   _feed_path = PROJECT_ROOT + _feed_path 
+  end
+  _feed_path = _feed_path.expand_path
+
+  before :all do
+    @feed_builder = Threatinator::FeedBuilder.from_file(_feed_path.to_s)
+  end
+
+  let(:feed_path) { feed_path.to_s }
+  let(:output_formatter) { Threatinator::Plugins::Output::Null.new(Threatinator::Plugins::Output::Null::Config.new) }
+  let(:feed_runner) { Threatinator::FeedRunner.new(feed, output_formatter) }
+  let(:feed) { @feed_builder.build() }
+end
+
+shared_examples_for 'any feed', :feed do
+  # Expects :provider, :name, :feed
+  subject { feed } 
+  it { should be_a(Threatinator::Feed) }
+
+  describe "#provider" do
+    subject { feed.provider }
+    it { is_expected.to be_a(::String) }
+    it { is_expected.to eq(provider) }
+  end
+
+  describe "#name" do
+    subject { feed.name}
+    it { is_expected.to be_a(::String) }
+    it { is_expected.to eq(name) }
+  end
+
+  describe "#parser_block" do
+    subject { feed.parser_block }
+    it { is_expected.to be_a(::Proc) }
+  end
+
+  describe "#fetcher_builder" do
+    subject { feed.fetcher_builder }
+    it { is_expected.to be_a ::Proc }
+    specify "when called, it should generate a kind of Threatinator::Fetcher" do
+      expect(subject.call).to be_kind_of(Threatinator::Fetcher)
+    end
+  end
+
+  describe "#parser_builder" do
+    subject { feed.parser_builder }
+    it { is_expected.to be_a ::Proc }
+
+    specify "when called, it should generate a kind of Threatinator::Parser" do
+      expect(subject.call).to be_kind_of(Threatinator::Parser)
+    end
+  end
+
+  describe "#filter_builders" do
+    subject { feed.filter_builders }
+
+    it "should be an Array of Proc objects" do
+      expect(subject).to be_an ::Array
+    end
+
+    specify "each Proc, when called, should generate an object that responds to :filter?" do
+      subject.each do |filter_builder|
+        expect(filter_builder.call).to respond_to(:filter?)
+      end
+    end
+  end
+
+  describe "#decoder_builders" do
+    subject { feed.decoder_builders }
+
+    it "should be an Array of Proc objects" do
+      expect(subject).to be_an ::Array
+    end
+
+    specify "each Proc, when called, should generate a kind of Threatinator::Decoder" do
+      subject.each do |decoder_builder|
+        expect(decoder_builder.call).to be_kind_of(Threatinator::Decoder)
+      end
+    end
+  end
+end
+
+module FeedHelpers
+  class FeedRunnerObserver
+    include Enumerable
+    attr_reader :records, :statuses, :events, :num_records_filtered, 
+      :num_records_missed, :num_records_parsed
+
+    def initialize
+      @records = []
+      @statuses = []
+      @events = []
+      @num_records_filtered = 0
+      @num_records_parsed = 0
+      @num_records_missed = 0
+    end
+
+    def each
+      @records.each_with_index do |record, i|
+        yield(record, @statuses[i], @events[i])
+      end
+    end
+
+    # Handles FeedRunner observations
+    def update(message, *args)
+      case message
+      when :record_missed
+        @records << args.shift
+        @statuses << :missed
+        @events << []
+        @num_records_missed += 1
+      when :record_filtered
+        @records << args.shift
+        @statuses << :filtered
+        @events << []
+        @num_records_filtered += 1
+      when :record_parsed
+        @records << args.shift
+        @statuses << :parsed
+        @events << args.shift
+        @num_records_parsed += 1
+      end
+    end
+  end
+
+  module FeedHelperMethods
+    def it_fetches_url(url)
+      describe 'fetching' do
+        it "should fetch the url #{url}" do
+          stub_request(:get, url)
+          feed.fetcher_builder.call().fetch()
+          expect(a_request(:get, url)).to have_been_made
+        end
+      end
+    end
+
+    def feed_data(filename)
+      (FEED_DATA_ROOT + filename).to_s
+    end
+
+    def describe_parsing_a_record(data, &block)
+      context("parsing a record from '#{data}'", :caller => caller) do
+        let(:observer) { FeedRunnerObserver.new }
+        before :each do
+          sio = StringIO.new(data)
+          feed_runner.add_observer(observer)
+          feed_runner.run(:io => sio)
+          @status, @record, @events = observer.first
+        end
+
+        after :each do
+          feed_runner.delete_observer(observer)
+        end
+
+        it "should have handled exactly 1 record" do
+          expect(observer.count).to eq(1)
+        end
+
+        describe "the record" do
+          include_context 'a parsed record'
+          instance_exec(&block)
+        end
+      end
+    end
+
+    def describe_parsing_the_file(filename, &block)
+      filepath = Pathname.new(filename)
+      relative_filename = filepath.relative_path_from(PROJECT_ROOT).to_s
+
+      context("parsing the file '#{relative_filename}'", :caller => caller) do
+        let(:observer) { FeedRunnerObserver.new }
+        before :each do
+          fio = File.open(filename, 'r')
+          feed_runner.add_observer(observer)
+          feed_runner.run(:io => fio)
+          fio.close unless fio.closed?
+        end
+
+        after :each do
+          feed_runner.delete_observer(observer)
+        end
+
+        include_context "a parsed feed"
+        instance_exec(&block)
+      end
+    end
+  end
+end

data/spec/support/shared/fetcher.rb

@@ -0,0 +1,48 @@
+require 'spec_helper'
+
+shared_examples_for "a fetcher" do
+  let(:expected_data) { "here's some data" }
+
+  subject { fetcher }
+  it { should respond_to(:fetch) }
+
+  it "should == itself" do
+    expect(fetcher).to be == fetcher
+  end
+
+  it "should == an identically configured instance" do
+    expect(fetcher_builder.call()).to be == fetcher_builder.call()
+  end
+
+  it "should not == a differently configured instance" do
+    expect(fetcher_builder.call()).not_to be == fetcher_builder_different.call()
+  end
+
+  it "should eql?(itself)" do
+    expect(fetcher).to eql(fetcher)
+  end
+
+  it "should eql? an identically configured instance" do
+    expect(fetcher_builder.call()).to eql(fetcher_builder.call())
+  end
+
+  it "should not eql? an differently configured instance" do
+    expect(fetcher_builder.call()).not_to eql(fetcher_builder_different.call())
+  end
+
+  context "the object returned by #fetch" do
+    subject { fetcher.fetch }
+
+    it_should_behave_like "an IO-like object"
+
+    it "should return the expected data when #read" do
+      expect(subject.read()).to eq(expected_data)
+    end
+  end
+end
+
+shared_examples_for "a #fetch call that failed" do
+  it "should raise Threatinator::Exceptions::FetchFailed" do
+    expect { fetcher.fetch() }.to raise_error(Threatinator::Exceptions::FetchFailed)
+  end
+end

data/spec/support/shared/filter.rb

@@ -0,0 +1,14 @@
+require 'spec_helper'
+
+shared_examples_for "a filter" do
+  # expects :filter, :should_filter, and :shouldnt_filter
+
+  describe "#filter?" do
+    it "should return true for data that is meant to be filtered" do
+      expect(filter.filter?(should_filter)).to eq(true)
+    end
+    it "should return false for data that is meant to be filtered" do
+      expect(filter.filter?(shouldnt_filter)).to eq(false)
+    end
+  end
+end

data/spec/support/shared/io-like.rb

@@ -0,0 +1,7 @@
+
+shared_examples_for "an IO-like object" do
+  it { is_expected.to respond_to(:close) }
+  it { is_expected.to respond_to(:read) }
+  it { is_expected.to respond_to(:gets) }
+  it { is_expected.to respond_to(:pos) }
+end

data/spec/support/shared/output.rb

@@ -0,0 +1,120 @@
+require 'spec_helper'
+require 'tempfile'
+require 'threatinator/plugin_loader'
+
+shared_examples_for "an output plugin" do |name|
+  let(:output) { described_class.new(config) }
+
+  specify "the plugin loader should find the class by (#{name.inspect})" do
+    loader = Threatinator::PluginLoader.new
+    loader.load_plugins(:output)
+    expect(loader.get(:output, name.to_sym)).to be(described_class)
+  end
+end
+
+shared_examples_for "a file-based output plugin" do |name|
+  it_should_behave_like "an output plugin", name
+
+  let(:event) { build(:event) }
+
+  describe "config.io => IO" do
+    let(:output) { described_class.new(config) }
+    let(:io) { StringIO.new }
+    before :each do
+      config.io = io
+    end
+
+    describe "#handle_event" do
+      before :each do
+        output.handle_event(event)
+      end
+
+      it "writes data to the provided IO" do
+        expect(io.string).not_to be_empty
+      end
+    end
+
+    describe "#finish" do
+      before :each do
+        allow(io).to receive(:close)
+        output.finish
+      end
+
+      it "closes the IO" do
+        expect(io).to have_received(:close)
+      end
+    end
+  end
+
+  describe "config.filename => String" do
+    let(:output) { described_class.new(config) }
+    before :each do
+      @tempfile = Tempfile.new("asdf")
+      @tempfile.close
+      @filepath = @tempfile.path
+      config.filename = @filepath
+    end
+
+    let(:filepath) { @filepath }
+
+    describe "#finish" do
+      it "closes the filehandle" do
+        expect(output.instance_variable_get(:"@output_io")).to receive(:close)
+        output.finish
+      end
+    end
+
+    describe "#handle_event" do
+      before :each do
+        output.handle_event(event)
+      end
+      specify "writes data to the path specified by :filename" do
+        output.finish
+        expect(File.read(@filepath)).not_to be_empty
+      end
+    end
+  end
+
+  describe "without specifying config.io or config.filename" do
+    before :each do
+      @orig_stdout = $stdout
+      @duped_io = StringIO.new
+      $stdout = double("stdout").as_null_object
+      allow($stdout).to receive(:dup).and_return(@duped_io)
+      @output = described_class.new(config)
+    end
+
+    let(:output) { @output }
+
+    after :each do
+      $stdout = @orig_stdout
+    end
+
+    it "uses a duplicate of $stdout" do
+      expect($stdout).to have_received(:dup)
+    end
+
+    describe "#handle_event" do
+      before :each do
+        output.handle_event(event)
+      end
+
+      it "writes to the dupe of $stdout" do
+        expect(@duped_io.string).not_to be_empty
+      end
+    end
+
+    describe "#finish" do
+      before :each do
+        allow(@duped_io).to receive(:close).and_call_original
+        output.finish
+      end
+
+      it "closes the dup of $stdout" do
+        expect(@duped_io).to have_received(:close)
+      end
+    end
+
+  end
+end
+