threatinator 0.1.6 → 0.2.0

data/spec/feeds/virbl-ip_reputation_spec.rb

@@ -0,0 +1,47 @@
+require 'spec_helper'
+
+describe 'feeds/virbl-ip_reputation.feed', :feed do
+  let(:provider) { 'virbl' }
+  let(:name) { 'ip_reputation' }
+
+  it_fetches_url 'http://virbl.org/download/virbl.dnsbl.bit.nl.txt'
+
+  describe_parsing_the_file feed_data('virbl-ip_reputation.txt') do
+    it "should have parsed 13 records" do
+      expect(num_records_parsed).to eq(13)
+    end
+    it "should have filtered 1 records" do
+      expect(num_records_filtered).to eq(1)
+    end
+  end
+
+  describe_parsing_a_record '193.20.147.167' do
+    it "should have parsed" do
+      expect(status).to eq(:parsed)
+    end
+    it "should have parsed 1 event" do
+      expect(events.count).to eq(1)
+    end
+    describe 'event 0' do
+      subject { events[0] }
+      its(:type) { is_expected.to be(:malware_host) }
+      its(:ipv4s) { is_expected.to  eq(build(:ipv4s, values: ['193.20.147.167'])) }
+    end
+  end
+
+  describe_parsing_a_record '37.84.148.198' do
+    it "should have parsed" do
+      expect(status).to eq(:parsed)
+    end
+    it "should have parsed 1 event" do
+      expect(events.count).to eq(1)
+    end
+    describe 'event 0' do
+      subject { events[0] }
+      its(:type) { is_expected.to be(:malware_host) }
+      its(:ipv4s) { is_expected.to  eq(build(:ipv4s, values: ['37.84.148.198'])) }
+    end
+  end
+end
+
+

data/spec/feeds/vxvault_url_reputation_spec.rb

@@ -0,0 +1,50 @@
+require 'spec_helper'
+
+describe 'feeds/vxvault-url_reputation.feed', :feed do
+  let(:provider) { 'vxvault' }
+  let(:name) { 'url_reputation' }
+
+  it_fetches_url 'http://vxvault.siri-urz.net/URL_List.php'
+
+  describe_parsing_the_file feed_data('vxvault-url-reputation.txt') do
+    it "should have parsed 12 records" do
+      expect(num_records_parsed).to eq(12)
+    end
+    it "should have filtered 3 records" do
+      expect(num_records_filtered).to eq(3)
+    end
+    it "should have missed 0 records" do
+      expect(num_records_missed).to eq(0)
+    end
+  end
+
+  describe_parsing_a_record 'http://176.31.228.6/111.exe' do
+    it "should have parsed" do
+      expect(status).to eq(:parsed)
+    end
+    it "should have parsed 1 event" do
+      expect(events.count).to eq(1)
+    end
+	describe 'event 0' do
+      subject { events[0] }
+      its(:type) { is_expected.to be(:malware_host) }
+	  its(:urls) { is_expected.to eq(build(:urls, values: ['http://176.31.228.6/111.exe'])) }
+    end
+  end
+
+  describe_parsing_a_record 'http://w719460.blob4.ge.tt/streams/2RLPNJy1/Hawlery.exe?sig=-Uidne-6-hjiAPHjRw-9FCmeKQculOB4naU&type=download' do
+    it "should have parsed" do
+      expect(status).to eq(:parsed)
+    end
+    it "should have parsed 1 event" do
+      expect(events.count).to eq(1)
+    end
+	describe 'event 0' do
+      subject { events[0] }
+      its(:type) { is_expected.to be(:malware_host) }
+	  its(:urls) { is_expected.to eq(build(:urls, values: ['http://w719460.blob4.ge.tt/streams/2RLPNJy1/Hawlery.exe?sig=-Uidne-6-hjiAPHjRw-9FCmeKQculOB4naU&type=download'])) }
+    end
+  end
+end
+
+

data/spec/feeds/yourcmc_ssh-ip_reputation_spec.rb

@@ -13,9 +13,6 @@ describe 'feeds/yourcmc_ssh-ip_reputation.feed', :feed do
     it "should have filtered 11 records" do
       expect(num_records_filtered).to eq(11)
     end
-    it "should have missed 0 records" do
-      expect(num_records_missed).to eq(0)
-    end
   end
 
   describe_parsing_a_record '109.120.157.63' do
@@ -28,7 +25,7 @@ describe 'feeds/yourcmc_ssh-ip_reputation.feed', :feed do
     describe 'event 0' do
       subject { events[0] }
       its(:type) { is_expected.to be(:scanning) }
-      its(:ipv4s) { is_expected.to match_array(['109.120.157.63']) }
+      its(:ipv4s) { is_expected.to  eq(build(:ipv4s, values: ['109.120.157.63'])) }
     end
   end
 
@@ -42,7 +39,7 @@ describe 'feeds/yourcmc_ssh-ip_reputation.feed', :feed do
     describe 'event 0' do
       subject { events[0] }
       its(:type) { is_expected.to be(:scanning) }
-      its(:ipv4s) { is_expected.to match_array(['109.165.15.113']) }
+      its(:ipv4s) { is_expected.to  eq(build(:ipv4s, values: ['109.165.15.113'])) }
     end
   end
 end

data/spec/feeds/yoyo_adservers_spec.rb

@@ -0,0 +1,47 @@
+require 'spec_helper'
+
+describe 'feeds/yoyo_adservers-domain_reputation.feed', :feed do
+  let(:provider) { 'yoyo' }
+  let(:name) { 'adservers' }
+
+  it_fetches_url 'http://pgl.yoyo.org/adservers/serverlist.php?hostformat=nohtml'
+
+  describe_parsing_the_file feed_data('yoyo_adservers.txt') do
+    it "should have parsed 25 records" do
+      expect(num_records_parsed).to eq(25)
+    end
+    it "should have filtered 0 records" do
+      expect(num_records_filtered).to eq(0)
+    end
+  end
+
+  describe_parsing_a_record '4affiliate.net' do
+    it "should have parsed" do
+      expect(status).to eq(:parsed)
+    end
+    it "should have parsed 1 event" do
+      expect(events.count).to eq(1)
+    end
+	describe 'event 0' do
+      subject { events[0] }
+      its(:type) { is_expected.to be(:scanning) }
+      its(:fqdns) { is_expected.to match_array(['4affiliate.net']) }
+    end
+  end
+
+  describe_parsing_a_record '600z.com' do
+    it "should have parsed" do
+      expect(status).to eq(:parsed)
+    end
+    it "should have parsed 1 event" do
+      expect(events.count).to eq(1)
+    end
+	describe 'event 0' do
+      subject { events[0] }
+      its(:type) { is_expected.to be(:scanning) }
+      its(:fqdns) { is_expected.to match_array(['600z.com']) }
+    end
+  end
+end
+
+

data/spec/feeds/zeus-domain_reputation_spec.rb

@@ -13,9 +13,6 @@ describe 'feeds/zeus-domain_reputation.feed', :feed do
     it "should have filtered 6 records" do
       expect(num_records_filtered).to eq(6)
     end
-    it "should have missed 0 records" do
-      expect(num_records_missed).to eq(0)
-    end
   end
 
   describe_parsing_a_record 'aconfideeeeeracia200.com' do

data/spec/feeds/zeus-ip_reputation_spec.rb

@@ -13,9 +13,6 @@ describe 'feeds/zeus-ip_reputation.feed', :feed do
     it "should have filtered 8 records" do
       expect(num_records_filtered).to eq(6)
     end
-    it "should have missed 0 records" do
-      expect(num_records_missed).to eq(0)
-    end
   end
 
   describe_parsing_a_record '109.229.210.250' do
@@ -28,7 +25,7 @@ describe 'feeds/zeus-ip_reputation.feed', :feed do
     describe 'event 0' do
       subject { events[0] }
       its(:type) { is_expected.to be(:c2) }
-      its(:ipv4s) { is_expected.to match_array(['109.229.210.250']) }
+      its(:ipv4s) { is_expected.to eq(build(:ipv4s, values: ['109.229.210.250'])) }
     end
   end
 
@@ -42,7 +39,7 @@ describe 'feeds/zeus-ip_reputation.feed', :feed do
     describe 'event 0' do
       subject { events[0] }
       its(:type) { is_expected.to be(:c2) }
-      its(:ipv4s) { is_expected.to match_array(['141.105.67.94']) }
+      its(:ipv4s) { is_expected.to eq(build(:ipv4s, values: ['141.105.67.94'])) }
     end
   end
 end

data/spec/spec_helper.rb

@@ -50,3 +50,5 @@ RSpec.configure do |config|
   end
 end
 
+require 'threatinator/logger'
+Threatinator::Logger.level = Threatinator::Logger::Levels::OFF

data/spec/support/factories/event.rb

@@ -1,4 +1,5 @@
 require 'threatinator/event'
+require 'threatinator/event_builder'
 
 FactoryGirl.define do
   factory :event, class: Threatinator::Event do
@@ -7,19 +8,22 @@ FactoryGirl.define do
     type :scanning
     ipv4s { [ ] }
     fqdns { [ ] }
+    urls  { [ ] }
 
     initialize_with { 
-      ret = new() 
-      ret.feed_name = feed_name
-      ret.feed_provider = feed_provider
-      ret.type = type
+      builder = Threatinator::EventBuilder.new(feed_provider, feed_name)
+      builder.type = type
+
       ipv4s.each do |ipv4|
-        ret.add_ipv4(ipv4)
+        builder.add_ipv4(ipv4)
       end
       fqdns.each do |fqdn|
-        ret.add_fqdn(fqdn)
+        builder.add_fqdn(fqdn)
+      end
+      urls.each do |url|
+        builder.add_url(url)
       end
-      ret
+      builder.build
     }
   end
 end

data/spec/support/factories/feed.rb

@@ -8,12 +8,39 @@ FactoryGirl.define do
     sequence(:provider) { |n| "provider_#{n}" }
     sequence(:name) { |n| "name_#{n}" }
     fetcher_builder { lambda { Threatinator::Fetcher.new({}) } } 
+    fetcher { nil }
     parser_builder { lambda { Threatinator::Parser.new({}) } } 
+    parser { nil }
     filter_builders { [] }
+    filters { [] }
     decoder_builders { [] }
+    decoders { [] }
     parser_block { lambda { |*args| } }
 
-    initialize_with { new(attributes) }
+    initialize_with { 
+      opts = attributes.to_hash
+      if fetcher = opts.delete(:fetcher)
+        opts[:fetcher_builder] = Proc.new { fetcher }
+      end
+      if decoders = opts.delete(:decoders)
+        decoders.each do |decoder|
+          opts[:decoder_builders] << Proc.new { decoder }
+        end
+      end
+      if filters = opts.delete(:filters)
+        filters.each do |filter|
+          if filter.kind_of?(::Proc)
+            filter = Threatinator::Filters::Block.new(filter)
+          end
+          fb = Proc.new { filter }
+          opts[:filter_builders] << fb
+        end
+      end
+      if parser = opts.delete(:parser)
+        opts[:parser_builder] = Proc.new { parser }
+      end
+      new(opts) 
+    }
 
     trait :http do
       url { "https://foobar/#{provider}/#{name}.data" }

data/spec/support/factories/ipv4.rb

@@ -0,0 +1,36 @@
+require 'threatinator/model/observables/ipv4'
+require 'ip'
+
+FactoryGirl.define do
+  factory :ipv4, class: Threatinator::Model::Observables::Ipv4 do
+    sequence(:ipv4) { |n| IP::V4.new(0xa000000 + n) } # Starts at 10.0.0.0
+
+    initialize_with do
+      opts = attributes.dup
+      if opts[:ipv4].is_a?(::String)
+        opts[:ipv4] = IP::V4.parse(opts[:ipv4])
+      end
+      new(opts)
+    end
+  end
+
+  factory :ipv4s, class: Threatinator::Model::Observables::Ipv4Collection do
+    values { [ ] }
+
+    initialize_with do
+      values = attributes[:values]
+
+      values.map! do |v|
+        if v.kind_of?(::String)
+          v = build(:ipv4, ipv4: v)
+        end
+        v
+      end
+
+      new(values)
+    end
+  end
+end
+
+
+

data/spec/support/factories/url.rb

@@ -0,0 +1,34 @@
+require 'threatinator/model/observables/url_collection'
+require 'addressable/uri'
+require 'ip'
+
+FactoryGirl.define do
+  factory :url, class: ::Addressable::URI do
+    url nil
+
+    initialize_with do
+      ::Addressable::URI.parse(attributes[:url])
+    end
+  end
+
+  factory :urls, class: Threatinator::Model::Observables::UrlCollection do
+    values { [ ] }
+
+    initialize_with do
+      values = attributes[:values]
+
+      values.map! do |v|
+        if v.kind_of?(::String)
+          v = build(:url, url: v)
+        end
+        v
+      end
+
+      new(values)
+    end
+  end
+end
+
+
+
+

data/spec/support/shared/feed_runner_observer.rb

@@ -0,0 +1,136 @@
+require 'spec_helper'
+require 'threatinator/feed_runner'
+
+shared_examples_for "a FeedRunner observer" do
+  describe "#update(:start)" do
+    after :each do
+      observer.update(:end)
+    end
+    it "does not raise any errors when handling a :start event" do
+      expect {
+        observer.update(:start)
+      }.not_to raise_error
+    end
+  end
+
+  describe "#update(:end)" do
+    before :each do
+      observer.update(:start)
+    end
+
+    it "does not raise any errors when handling an :end event" do
+      expect {
+        observer.update(:end)
+      }.not_to raise_error
+    end
+  end
+
+  context "once started" do
+    before :each do
+      observer.update(:start)
+    end
+
+    after :each do
+      observer.update(:end)
+    end
+
+    let(:record) { build(:record, line_number: 23, pos_start: 99, pos_end: 105, data: "foobar\r\n") }
+
+    describe "#update(:start_fetch)" do
+      it "does not raise any errors when handling an :start_fetch event" do
+        expect {
+          observer.update(:start_fetch)
+        }.not_to raise_error
+      end
+    end
+
+    describe "#update(:end_fetch)" do
+      it "does not raise any errors when handling an :end_fetch event" do
+        expect {
+          observer.update(:end_fetch)
+        }.not_to raise_error
+      end
+    end
+
+    describe "#update(:start_decode)" do
+      it "does not raise any errors when handling an :start_decode event" do
+        expect {
+          observer.update(:start_decode)
+        }.not_to raise_error
+      end
+    end
+
+    describe "#update(:end_decode)" do
+      it "does not raise any errors when handling an :end_decode event" do
+        expect {
+          observer.update(:end_decode)
+        }.not_to raise_error
+      end
+    end
+
+    describe "#update(:start_parse_record, record)" do
+      it "does not raise any errors when handling an :start_parse_record event" do
+        expect {
+          observer.update(:start_parse_record, record)
+        }.not_to raise_error
+      end
+    end
+
+    describe "#update(:end_parse_record, record)" do
+      it "does not raise any errors when handling an :end_parse_record event" do
+        expect {
+          observer.update(:end_parse_record, record)
+        }.not_to raise_error
+      end
+    end
+
+    describe "#update(:record_filtered, record)" do
+      it "does not raise any errors when handling a :record_filtered event" do
+        expect {
+          observer.update(:record_filtered, record)
+        }.not_to raise_error
+      end
+    end
+
+    describe "#update(:record_missed, record)" do
+      it "does not raise any errors when handling a :record_missed event" do
+        expect {
+          observer.update(:record_missed, record)
+        }.not_to raise_error
+      end
+    end
+
+    describe "#update(:record_parsed, record, events)" do
+      it "does not raise any errors when handling a :record_parsed event" do
+        expect {
+          observer.update(:record_missed, record, [build(:event)])
+        }.not_to raise_error
+      end
+    end
+
+    describe "#update(:record_error, record, array_of_errors)" do
+      it "does not raise any errors when handling a :record_error event" do
+        errors = [
+          Threatinator::Exceptions::EventBuildError.new("error 1"),
+          Threatinator::Exceptions::EventBuildError.new("error 2"),
+          Threatinator::Exceptions::EventBuildError.new("error 3")
+        ]
+        expect {
+          observer.update(:record_missed, record, errors)
+        }.not_to raise_error
+      end
+    end
+
+    it "does not raise any errors when handling unknown messages" do
+      expect {
+        observer.update(:flibby_floo, record)
+      }.not_to raise_error
+    end
+
+    it "does not raise any errors when handling extra arguments" do
+      expect {
+        observer.update(:flibby_floo, record, 1, 2, 3, 4)
+      }.not_to raise_error
+    end
+  end
+end