shadowbq-threatinator 0.5.0

data/spec/support/bad_feeds/missing_parser.feed

@@ -0,0 +1,3 @@
+provider "provider1"
+name "name1"
+fetch_http('https://foobar/feed1.data')

data/spec/support/bad_feeds/missing_provider.feed

@@ -0,0 +1,5 @@
+name "feed1"
+fetch_http('https://foobar/feed1.data')
+
+parse_eachline(:separator => "\n") do |builder, line|
+end

data/spec/support/factories/event.rb

@@ -0,0 +1,31 @@
+require 'threatinator/event'
+require 'threatinator/event_builder'
+
+FactoryGirl.define do
+  factory :event, class: Threatinator::Event do
+    feed_name 'my_feed_name'
+    feed_provider 'my_provider'
+    type :scanning
+    ipv4s { [ ] }
+    fqdns { [ ] }
+    urls  { [ ] }
+
+    initialize_with { 
+      builder = Threatinator::EventBuilder.new(feed_provider, feed_name)
+      builder.type = type
+
+      ipv4s.each do |ipv4|
+        builder.add_ipv4(ipv4)
+      end
+      fqdns.each do |fqdn|
+        builder.add_fqdn(fqdn)
+      end
+      urls.each do |url|
+        builder.add_url(url)
+      end
+      builder.build
+    }
+  end
+end
+
+

data/spec/support/factories/feed.rb

@@ -0,0 +1,59 @@
+require 'threatinator/feed'
+require 'threatinator/parser'
+require 'threatinator/fetcher'
+require 'threatinator/fetchers/http'
+
+FactoryGirl.define do
+  factory :feed, class: Threatinator::Feed do
+    sequence(:provider) { |n| "provider_#{n}" }
+    sequence(:name) { |n| "name_#{n}" }
+    fetcher_builder { lambda { Threatinator::Fetcher.new({}) } } 
+    fetcher { nil }
+    parser_builder { lambda { Threatinator::Parser.new({}) } } 
+    parser { nil }
+    filter_builders { [] }
+    filters { [] }
+    decoder_builders { [] }
+    decoders { [] }
+    parser_block { lambda { |*args| } }
+
+    initialize_with { 
+      opts = attributes.to_hash
+      if fetcher = opts.delete(:fetcher)
+        opts[:fetcher_builder] = Proc.new { fetcher }
+      end
+      if decoders = opts.delete(:decoders)
+        decoders.each do |decoder|
+          opts[:decoder_builders] << Proc.new { decoder }
+        end
+      end
+      if filters = opts.delete(:filters)
+        filters.each do |filter|
+          if filter.kind_of?(::Proc)
+            filter = Threatinator::Filters::Block.new(filter)
+          end
+          fb = Proc.new { filter }
+          opts[:filter_builders] << fb
+        end
+      end
+      if parser = opts.delete(:parser)
+        opts[:parser_builder] = Proc.new { parser }
+      end
+      new(opts) 
+    }
+
+    trait :http do
+      url { "https://foobar/#{provider}/#{name}.data" }
+      fetcher_builder { lambda { Threatinator::Fetchers::Http.new({url: url}) } } 
+    end
+
+    trait :mini do
+      http
+      sequence(:url) { |n| "http://x#{n}" }
+      sequence(:provider) { |n| "x#{n}" }
+      sequence(:name) { |n| "x#{n}" }
+    end
+  end
+
+end
+

data/spec/support/factories/feed_builder.rb

@@ -0,0 +1,65 @@
+require 'threatinator/feed_builder'
+
+FactoryGirl.define do
+  factory :feed_builder, class: Threatinator::FeedBuilder do
+    initialize_with do
+      builder = new()
+      attributes.each_pair do |sym, val|
+        next if val.nil?
+        if val.kind_of?(::Proc)
+          builder.send(sym, &val)
+        else 
+          builder.send(sym, val)
+        end
+      end
+      builder
+    end
+
+    trait :provider do
+      provider 'FakeSecureCo'
+    end
+
+    trait :name do
+      name 'MaliciousDataFeed'
+    end
+
+    trait :http do
+      fetch_http "http://foo.com/bar"
+    end
+
+    trait :parse_eachline do
+      parse_eachline { lambda { |line|  } } 
+    end
+
+    trait :without_provider do
+      name
+      parse_eachline
+      http
+    end
+
+    trait :without_name do
+      provider
+      parse_eachline
+      http
+    end
+
+    trait :without_parser do
+      name
+      provider
+      http
+    end
+
+    trait :without_fetcher do
+      name
+      provider
+      parse_eachline
+    end
+
+    trait :buildable do
+      name
+      provider
+      parse_eachline
+      http
+    end
+  end
+end

data/spec/support/factories/feed_registry.rb

@@ -0,0 +1,8 @@
+require 'threatinator/feed_registry'
+
+FactoryGirl.define do
+  factory :feed_registry, class: Threatinator::FeedRegistry do
+  end
+end
+
+

data/spec/support/factories/ipv4.rb

@@ -0,0 +1,36 @@
+require 'threatinator/model/observables/ipv4'
+require 'ip'
+
+FactoryGirl.define do
+  factory :ipv4, class: Threatinator::Model::Observables::Ipv4 do
+    sequence(:ipv4) { |n| IP::V4.new(0xa000000 + n) } # Starts at 10.0.0.0
+
+    initialize_with do
+      opts = attributes.dup
+      if opts[:ipv4].is_a?(::String)
+        opts[:ipv4] = IP::V4.parse(opts[:ipv4])
+      end
+      new(opts)
+    end
+  end
+
+  factory :ipv4s, class: Threatinator::Model::Observables::Ipv4Collection do
+    values { [ ] }
+
+    initialize_with do
+      values = attributes[:values]
+
+      values.map! do |v|
+        if v.kind_of?(::String)
+          v = build(:ipv4, ipv4: v)
+        end
+        v
+      end
+
+      new(values)
+    end
+  end
+end
+
+
+

data/spec/support/factories/output.rb

@@ -0,0 +1,11 @@
+require 'threatinator/output'
+
+FactoryGirl.define do
+  sequence :output_name do |n|
+    name = "output_test#{n}"
+    name.to_sym
+  end
+end
+
+
+

data/spec/support/factories/record.rb

@@ -0,0 +1,17 @@
+require 'threatinator/record'
+
+FactoryGirl.define do
+  factory :record, class: Threatinator::Record do
+    data { "some data" }
+
+    line_number 1
+    pos_start 0
+    pos_end 9
+
+    initialize_with { 
+      new(attributes[:data], attributes) 
+    }
+  end
+end
+
+

data/spec/support/factories/url.rb

@@ -0,0 +1,34 @@
+require 'threatinator/model/observables/url_collection'
+require 'addressable/uri'
+require 'ip'
+
+FactoryGirl.define do
+  factory :url, class: ::Addressable::URI do
+    url nil
+
+    initialize_with do
+      ::Addressable::URI.parse(attributes[:url])
+    end
+  end
+
+  factory :urls, class: Threatinator::Model::Observables::UrlCollection do
+    values { [ ] }
+
+    initialize_with do
+      values = attributes[:values]
+
+      values.map! do |v|
+        if v.kind_of?(::String)
+          v = build(:url, url: v)
+        end
+        v
+      end
+
+      new(values)
+    end
+  end
+end
+
+
+
+

data/spec/support/factories/xml_node.rb

@@ -0,0 +1,33 @@
+require 'threatinator/parsers/xml/node'
+
+FactoryGirl.define do
+  factory :xml_node, class: Threatinator::Parsers::XML::Node do
+    name { "foo" }
+    attrs { { } }
+    text { "" }
+    children { [] }
+
+    initialize_with { 
+      new(name, attrs: attrs, text: text, children: children) 
+    }
+
+    trait(:with_attrs) do
+      attrs { {
+        attr1: "val1",
+        attr2: "val2"
+      } }
+    end
+
+    trait(:with_children) do
+      children { [
+        build(:xml_node, name: "child1"),
+        build(:xml_node, name: "child2"),
+        build(:xml_node, name: "child3"),
+      ] }
+    end
+  end
+
+end
+
+
+

data/spec/support/helpers/io.rb

@@ -0,0 +1,11 @@
+require 'stringio'
+module IOHelpers
+  def temp_stdout
+    $stdout = StringIO.new
+    yield $stdout.string
+    return $stdout.string
+  ensure
+    $stdout = STDOUT
+  end
+end
+

data/spec/support/helpers/models.rb

@@ -0,0 +1,13 @@
+require 'threatinator/parser'
+require 'threatinator/fetcher'
+
+module FeedSpec
+  class Fetcher < Threatinator::Fetcher
+    def initialize(opts = {})
+      @io = opts[:io]
+    end
+    def fetch; @io; end
+  end
+  class Parser < Threatinator::Parser
+  end
+end

data/spec/support/shared/action_builder.rb

@@ -0,0 +1,47 @@
+require 'spec_helper'
+
+shared_examples_for "an action builder" do
+  # expects :builder
+  # expects :config_hash
+  describe "a call to #feed_registry" do
+    let(:feed_registry) { double('feed_registry') }
+    let(:feed_search_hash) { double('feed search hash') }
+    let(:feed_search) { double('feed_search') }
+
+    before :each do
+      allow(Threatinator::FeedRegistry).to receive(:build).and_return(feed_registry)
+      allow(Threatinator::Config::FeedSearch).to receive(:new).and_return(feed_search)
+    end
+
+
+    context "when config_hash['feed_search'] exists" do
+      before :each do
+        config_hash['feed_search'] = feed_search_hash
+      end
+
+      it "builds a new Threatinator::Config::FeedSearch using config_hash['feed_search']" do
+        expect(Threatinator::Config::FeedSearch).to receive(:new).with(feed_search_hash)
+        builder.feed_registry
+      end
+    end
+    context "when config_hash['feed_search'] does not exist" do
+      before :each do
+        config_hash.delete('feed_search')
+      end
+
+      it "builds a new Threatinator::Config::FeedSearch using an empty hash" do
+        expect(Threatinator::Config::FeedSearch).to receive(:new).with({})
+        builder.feed_registry
+      end
+    end
+
+    it "builds a new feed registry using the config" do
+      expect(Threatinator::FeedRegistry).to receive(:build).with(feed_search)
+      builder.feed_registry
+    end
+
+    it "returns the instance of Threatinator::FeedRegistry" do
+      expect(builder.feed_registry).to be(feed_registry)
+    end
+  end
+end

data/spec/support/shared/decoder.rb

@@ -0,0 +1,70 @@
+# encoding: utf-8
+require 'spec_helper'
+require 'threatinator/exceptions'
+
+shared_examples_for "a decoder" do
+  # Expects :encode_data_proc, :decoder_opts
+  let(:extra_opts) { { } }
+  let(:decoder) { described_class.new(decoder_opts.merge(extra_opts)) }
+
+  describe "an instance" do
+    subject { decoder }
+    it { is_expected.to respond_to(:decode) }
+
+    it "should close the IO that it decodes from" do
+      data = encode_data_proc.call("here's some data")
+      io = StringIO.new(data)
+      expect(io).not_to be_closed
+      decoder.decode(io)
+      expect(io).to be_closed
+    end
+
+  end
+
+  describe "decoding a UTF-8 string" do 
+    let(:original_string) { 
+      "\xE1\x9A\xA0\xE1\x9B\x87\xE1\x9A\xBB\xE1\x9B\xAB\xE1\x9B\x92\xE1\x9B\xA6\xE1\x9A\xA6\xE1\x9B\xAB\xE1\x9A\xA0\xE1\x9A\xB1\xE1\x9A\xA9\xE1\x9A\xA0\xE1\x9A\xA2\xE1\x9A\xB1\xE1\x9B\xAB\xE1\x9A\xA0\xE1\x9B\x81\xE1\x9A\xB1\xE1\x9A\xAA\xE1\x9B\xAB\xE1\x9A\xB7\xE1\x9B\x96\xE1\x9A\xBB\xE1\x9A\xB9\xE1\x9B\xA6\xE1\x9B\x9A\xE1\x9A\xB3\xE1\x9A\xA2\xE1\x9B\x97\n\xE1\x9B\x8B\xE1\x9A\xB3\xE1\x9B\x96\xE1\x9A\xAA\xE1\x9B\x9A\xE1\x9B\xAB\xE1\x9A\xA6\xE1\x9B\x96\xE1\x9A\xAA\xE1\x9A\xBB\xE1\x9B\xAB\xE1\x9B\x97\xE1\x9A\xAA\xE1\x9A\xBE\xE1\x9A\xBE\xE1\x9A\xAA\xE1\x9B\xAB\xE1\x9A\xB7\xE1\x9B\x96\xE1\x9A\xBB\xE1\x9A\xB9\xE1\x9B\xA6\xE1\x9B\x9A\xE1\x9A\xB3\xE1\x9B\xAB\xE1\x9B\x97\xE1\x9B\x81\xE1\x9A\xB3\xE1\x9B\x9A\xE1\x9A\xA2\xE1\x9A\xBE\xE1\x9B\xAB\xE1\x9A\xBB\xE1\x9B\xA6\xE1\x9B\x8F\xE1\x9B\xAB\xE1\x9B\x9E\xE1\x9A\xAB\xE1\x9B\x9A\xE1\x9A\xAA\xE1\x9A\xBE\n\xE1\x9A\xB7\xE1\x9B\x81\xE1\x9A\xA0\xE1\x9B\xAB\xE1\x9A\xBB\xE1\x9B\x96\xE1\x9B\xAB\xE1\x9A\xB9\xE1\x9B\x81\xE1\x9B\x9A\xE1\x9B\x96\xE1\x9B\xAB\xE1\x9A\xA0\xE1\x9A\xA9\xE1\x9A\xB1\xE1\x9B\xAB\xE1\x9B\x9E\xE1\x9A\xB1\xE1\x9B\x81\xE1\x9A\xBB\xE1\x9B\x8F\xE1\x9A\xBE\xE1\x9B\x96\xE1\x9B\xAB\xE1\x9B\x9E\xE1\x9A\xA9\xE1\x9B\x97\xE1\x9B\x96\xE1\x9B\x8B\xE1\x9B\xAB\xE1\x9A\xBB\xE1\x9B\x9A\xE1\x9B\x87\xE1\x9B\x8F\xE1\x9A\xAA\xE1\x9A\xBE\xE1\x9B\xAC\n"
+      .force_encoding("UTF-8")
+    }
+
+    let(:encoded_string) { encode_data_proc.call(original_string) }
+    let(:encoded_io) { StringIO.new(encoded_string) }
+
+    describe "the decoded data" do
+      it "should equal the original string" do
+        expect(decoder.decode(encoded_io).read()).to eq(original_string)
+      end
+      it "should be UTF-8 encoded" do
+        expect(decoder.decode(encoded_io).read().encoding).to eq(Encoding::UTF_8)
+      end
+    end
+  end
+
+  describe "decoding a UTF-8 string2" do 
+    let(:original_string) { 
+      "21826        |  Corporación Telemic C.A.,VE    |    200.75.105.49  |  2014-07-18 19:54:54  |  sshpwauth".force_encoding("UTF-8")
+    }
+
+    let(:encoded_string) { encode_data_proc.call(original_string) }
+    let(:encoded_io) { StringIO.new(encoded_string) }
+
+    describe "the decoded data" do
+      it "should equal the original string" do
+        expect(decoder.decode(encoded_io).read()).to eq(original_string)
+      end
+      it "should be UTF-8 encoded" do
+        expect(decoder.decode(encoded_io).read().encoding).to eq(Encoding::UTF_8)
+      end
+    end
+  end
+end
+
+shared_examples_for "a decoder encountering an error during decoding" do
+  # Expects :decoder, :input_io
+  it "#decode should raise a DecoderError" do
+    expect { 
+      decoder.decode(input_io)
+    }.to raise_error(Threatinator::Exceptions::DecoderError)
+  end
+end
+

data/spec/support/shared/feed_runner_observer.rb

@@ -0,0 +1,136 @@
+require 'spec_helper'
+require 'threatinator/feed_runner'
+
+shared_examples_for "a FeedRunner observer" do
+  describe "#update(:start)" do
+    after :each do
+      observer.update(:end)
+    end
+    it "does not raise any errors when handling a :start event" do
+      expect {
+        observer.update(:start)
+      }.not_to raise_error
+    end
+  end
+
+  describe "#update(:end)" do
+    before :each do
+      observer.update(:start)
+    end
+
+    it "does not raise any errors when handling an :end event" do
+      expect {
+        observer.update(:end)
+      }.not_to raise_error
+    end
+  end
+
+  context "once started" do
+    before :each do
+      observer.update(:start)
+    end
+
+    after :each do
+      observer.update(:end)
+    end
+
+    let(:record) { build(:record, line_number: 23, pos_start: 99, pos_end: 105, data: "foobar\r\n") }
+
+    describe "#update(:start_fetch)" do
+      it "does not raise any errors when handling an :start_fetch event" do
+        expect {
+          observer.update(:start_fetch)
+        }.not_to raise_error
+      end
+    end
+
+    describe "#update(:end_fetch)" do
+      it "does not raise any errors when handling an :end_fetch event" do
+        expect {
+          observer.update(:end_fetch)
+        }.not_to raise_error
+      end
+    end
+
+    describe "#update(:start_decode)" do
+      it "does not raise any errors when handling an :start_decode event" do
+        expect {
+          observer.update(:start_decode)
+        }.not_to raise_error
+      end
+    end
+
+    describe "#update(:end_decode)" do
+      it "does not raise any errors when handling an :end_decode event" do
+        expect {
+          observer.update(:end_decode)
+        }.not_to raise_error
+      end
+    end
+
+    describe "#update(:start_parse_record, record)" do
+      it "does not raise any errors when handling an :start_parse_record event" do
+        expect {
+          observer.update(:start_parse_record, record)
+        }.not_to raise_error
+      end
+    end
+
+    describe "#update(:end_parse_record, record)" do
+      it "does not raise any errors when handling an :end_parse_record event" do
+        expect {
+          observer.update(:end_parse_record, record)
+        }.not_to raise_error
+      end
+    end
+
+    describe "#update(:record_filtered, record)" do
+      it "does not raise any errors when handling a :record_filtered event" do
+        expect {
+          observer.update(:record_filtered, record)
+        }.not_to raise_error
+      end
+    end
+
+    describe "#update(:record_missed, record)" do
+      it "does not raise any errors when handling a :record_missed event" do
+        expect {
+          observer.update(:record_missed, record)
+        }.not_to raise_error
+      end
+    end
+
+    describe "#update(:record_parsed, record, events)" do
+      it "does not raise any errors when handling a :record_parsed event" do
+        expect {
+          observer.update(:record_missed, record, [build(:event)])
+        }.not_to raise_error
+      end
+    end
+
+    describe "#update(:record_error, record, array_of_errors)" do
+      it "does not raise any errors when handling a :record_error event" do
+        errors = [
+          Threatinator::Exceptions::EventBuildError.new("error 1"),
+          Threatinator::Exceptions::EventBuildError.new("error 2"),
+          Threatinator::Exceptions::EventBuildError.new("error 3")
+        ]
+        expect {
+          observer.update(:record_missed, record, errors)
+        }.not_to raise_error
+      end
+    end
+
+    it "does not raise any errors when handling unknown messages" do
+      expect {
+        observer.update(:flibby_floo, record)
+      }.not_to raise_error
+    end
+
+    it "does not raise any errors when handling extra arguments" do
+      expect {
+        observer.update(:flibby_floo, record, 1, 2, 3, 4)
+      }.not_to raise_error
+    end
+  end
+end