shadowbq-threatinator 0.5.0

data/lib/threatinator/registry.rb

@@ -0,0 +1,53 @@
+require 'threatinator/exceptions'
+
+module Threatinator
+  # Just a simple class that holds stuff. Yup, a glorified hash.
+  class Registry
+    include Threatinator::Exceptions
+
+    def initialize()
+      @data= Hash.new
+    end
+
+    # @param [Object] key The object to use as the key for storing the object
+    # @param [Object] object The object to be stored
+    # @raise [Threatinator::Exceptions::dAlreadyRegisteredError] if an object
+    #  with the same key is already registered.
+    def register(key, object)
+      if @data.has_key?(key)
+        raise AlreadyRegisteredError.new(key)
+      end
+      @data[key] = object
+    end
+
+    # @param [Object] key 
+    # @return [Object]
+    def get(key)
+      @data[key]
+    end
+
+    # @return [Array<Object>] an array of keys
+    def keys
+      @data.keys
+    end
+
+    # @return [Integer] the number of objects in the registry
+    def count
+      @data.count
+    end
+
+    # Enumerates through each object in our registry
+    # @yield [object]
+    # @yieldparam [Object] object An object within the registry
+    def each(&block)
+      return enum_for(:each) unless block_given?
+      @data.each_pair(&block)
+    end
+
+    # Removes all objects from the registry
+    def clear
+      @data.clear
+    end
+  end
+end
+

data/lib/threatinator/util.rb

@@ -0,0 +1,15 @@
+
+module Threatinator
+  module Util
+    def underscore2cc(str)
+      str.to_s.split('_').map {|e| e.capitalize }.join
+    end
+    module_function :underscore2cc
+
+    def cc2underscore(str)
+      str.to_s.split('_').map {|e| e.capitalize }.join
+    end
+    module_function :underscore2cc
+  end
+end
+

data/lib/threatinator.rb

@@ -0,0 +1,3 @@
+module Threatinator
+  require 'threatinator/runner'
+end

data/spec/feeds/ET_block-ip_reputation_spec.rb

@@ -0,0 +1,50 @@
+require 'spec_helper'
+
+describe 'feeds/ET_block-ip_reputation.feed', :feed do
+  let(:provider) { 'emergingthreats' }
+  let(:name) { 'block_ip_reputation' }
+  let(:event_types) { [:scanning]}
+
+
+  it_fetches_url 'http://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt'
+
+  describe_parsing_the_file feed_data('ET_block-ip_reputation.txt') do
+    it "should have parsed 12 records" do
+      expect(num_records_parsed).to eq(12)
+    end
+    it "should have filtered 68 records" do
+      expect(num_records_filtered).to eq(68)
+    end
+    it "should have missed 0 records" do
+      expect(num_records_missed).to eq(0)
+    end
+  end
+
+  describe_parsing_a_record '103.230.84.239' do
+    it "should have parsed" do
+      expect(status).to eq(:parsed)
+    end
+    it "should have parsed 1 event" do
+      expect(events.count).to eq(1)
+    end
+	describe 'event 0' do
+      subject { events[0] }
+      its(:type) { is_expected.to be(:c2) }
+	  its(:ipv4s) { is_expected.to eq(build(:ipv4s, values: ['103.230.84.239'])) }
+    end
+  end
+
+  describe_parsing_a_record '97.107.134.249' do
+    it "should have parsed" do
+      expect(status).to eq(:parsed)
+    end
+    it "should have parsed 1 event" do
+      expect(events.count).to eq(1)
+    end
+	describe 'event 0' do
+      subject { events[0] }
+      its(:type) { is_expected.to be(:c2) }
+	  its(:ipv4s) { is_expected.to eq(build(:ipv4s, values: ['97.107.134.249'])) }
+    end
+  end
+end

data/spec/feeds/ET_compromised-ip_reputation_spec.rb

@@ -0,0 +1,47 @@
+require 'spec_helper'
+
+describe 'feeds/ET_compromised-ip_reputation.feed', :feed do
+  let(:provider) { 'emergingthreats' }
+  let(:name) { 'compromised_ip_reputation' }
+  let(:event_types) { [:scanning]}
+
+
+  it_fetches_url 'http://rules.emergingthreats.net/open/suricata/rules/compromised-ips.txt'
+
+  describe_parsing_the_file feed_data('ET_compromised-ip_reputation.txt') do
+    it "should have parsed 11 records" do
+      expect(num_records_parsed).to eq(11)
+    end
+    it "should have filtered 0 records" do
+      expect(num_records_filtered).to eq(0)
+    end
+  end
+
+  describe_parsing_a_record '1.93.24.90' do
+    it "should have parsed" do
+      expect(status).to eq(:parsed)
+    end
+    it "should have parsed 1 event" do
+      expect(events.count).to eq(1)
+    end
+	describe 'event 0' do
+      subject { events[0] }
+      its(:type) { is_expected.to be(:scanning) }
+      its(:ipv4s) { is_expected.to  eq(build(:ipv4s, values: ['1.93.24.90'])) }
+    end
+  end
+
+  describe_parsing_a_record '1.93.26.32' do
+    it "should have parsed" do
+      expect(status).to eq(:parsed)
+    end
+    it "should have parsed 1 event" do
+      expect(events.count).to eq(1)
+    end
+	describe 'event 0' do
+      subject { events[0] }
+      its(:type) { is_expected.to be(:scanning) }
+      its(:ipv4s) { is_expected.to  eq(build(:ipv4s, values: ['1.93.26.32'])) }
+    end
+  end
+end

data/spec/feeds/ET_openbadlist-ip_reputation_spec.rb

@@ -0,0 +1,56 @@
+require 'spec_helper'
+
+describe 'feeds/ET_openbadlist-ip_reputation.feed', :feed do
+  let(:provider) { 'emergingthreats' }
+  let(:name) { 'openbadlist_ip_reputation' }
+  let(:event_types) { [:scanning]}
+
+
+  it_fetches_url 'https://raw.githubusercontent.com/EmergingThreats/et-open-bad-ip-list/master/IPs.txt'
+
+  describe_parsing_the_file feed_data('ET_openbadlist-ip_reputation.txt') do
+    it "should have parsed 44 records" do
+      expect(num_records_parsed).to eq(44)
+    end
+    it "should have filtered 18 records" do
+      expect(num_records_filtered).to eq(18)
+    end
+  end
+
+
+  describe_parsing_a_record 'Feb 18 2014; 89.45.14.0/24; Infinity/Redkit2/Goon EK landing or EK gate.' do
+    it "should have been filtered" do
+      expect(status).to eq(:filtered)
+    end
+    it "should have parsed 0 events" do
+      expect(events.count).to eq(0)
+    end
+  end
+  describe_parsing_a_record 'Jan 24 2014; 212.83.160.187/32; Neutrino EK' do
+    it "should have parsed" do
+      expect(status).to eq(:parsed)
+    end
+    it "should have parsed 1 event" do
+      expect(events.count).to eq(1)
+    end
+	describe 'event 0' do
+      subject { events[0] }
+      its(:type) { is_expected.to be(:c2) }
+      its(:ipv4s) { is_expected.to eq(build(:ipv4s, values: ['212.83.160.187'])) }
+    end
+  end
+
+  describe_parsing_a_record 'Jan 28 2014; 149.154.64.180/32; [a-z]{3,6}\.pp\.ua DGA cesspool' do
+    it "should have parsed" do
+      expect(status).to eq(:parsed)
+    end
+    it "should have parsed 1 event" do
+      expect(events.count).to eq(1)
+    end
+	describe 'event 0' do
+      subject { events[0] }
+      its(:type) { is_expected.to be(:c2) }
+      its(:ipv4s) { is_expected.to eq(build(:ipv4s, values: ['149.154.64.180'])) }
+    end
+  end
+end

data/spec/feeds/alienvault-ip_reputation_spec.rb

@@ -0,0 +1,46 @@
+require 'spec_helper'
+
+describe 'feeds/alienvault-ip_reputation.feed', :feed do
+  let(:provider) { 'alienvault' }
+  let(:name) { 'ip_reputation' }
+  let(:event_types) { [:scanning]}
+  
+  it_fetches_url 'https://reputation.alienvault.com/reputation.generic'
+
+  describe_parsing_the_file feed_data('alienvault-ip_reputation.txt') do
+    it "should have parsed 10 records" do
+      expect(num_records_parsed).to eq(10)
+    end
+    it "should have filtered 8 records" do
+      expect(num_records_filtered).to eq(8)
+    end
+  end
+
+  describe_parsing_a_record '37.205.198.162 # Scanning Host IT,,42.8333015442,12.8332996368' do
+    it "should have parsed" do
+      expect(status).to eq(:parsed)
+    end
+    it "should have parsed 1 event" do
+      expect(events.count).to eq(1)
+    end
+    describe 'event 0' do
+      subject { events[0] }
+      its(:type) { is_expected.to be(:scanning) }
+      its(:ipv4s) { is_expected.to  eq(build(:ipv4s, values: ['37.205.198.162'])) }
+    end
+  end
+
+  describe_parsing_a_record '108.59.1.5 # Scanning Host A1,,0.0,0.0' do
+    it "should have parsed" do
+      expect(status).to eq(:parsed)
+    end
+    it "should have parsed 1 event" do
+      expect(events.count).to eq(1)
+    end
+    describe 'event 0' do
+      subject { events[0] }
+      its(:type) { is_expected.to be(:scanning) }
+      its(:ipv4s) { is_expected.to  eq(build(:ipv4s, values: ['108.59.1.5'])) }
+    end
+  end
+end

data/spec/feeds/arbor_fastflux-domain_reputation_spec.rb

@@ -0,0 +1,46 @@
+require 'spec_helper'
+
+describe 'feeds/arbor_fastflux-domain_reputation.feed', :feed do
+  let(:provider) { 'arbor' }
+  let(:name) { 'fastflux_domain_reputation' }
+  let(:event_types) { [:scanning]}
+  
+  it_fetches_url 'http://atlas.arbor.net/summary/domainlist'
+
+  describe_parsing_the_file feed_data('arbor_domainlist.txt') do
+    it "should have parsed 2 records" do
+      expect(num_records_parsed).to eq(2)
+    end
+    it "should have filtered 9 records" do
+      expect(num_records_filtered).to eq(9)
+    end
+  end
+
+  describe_parsing_a_record 'brylanehome.com' do
+    it "should have parsed" do
+      expect(status).to eq(:parsed)
+    end
+    it "should have parsed 1 event" do
+      expect(events.count).to eq(1)
+    end
+	describe 'event 0' do
+      subject { events[0] }
+      its(:type) { is_expected.to be(:c2) }
+      its(:fqdns) { is_expected.to match_array(['brylanehome.com']) }
+    end
+  end
+
+  describe_parsing_a_record 'emltrk.com' do
+    it "should have parsed" do
+      expect(status).to eq(:parsed)
+    end
+    it "should have parsed 1 event" do
+      expect(events.count).to eq(1)
+    end
+	describe 'event 0' do
+      subject { events[0] }
+      its(:type) { is_expected.to be(:c2) }
+      its(:fqdns) { is_expected.to match_array(['emltrk.com']) }
+    end
+  end
+end

data/spec/feeds/arbor_ssh-ip_reputation_spec.rb

@@ -0,0 +1,46 @@
+require 'spec_helper'
+
+describe 'feeds/arbor_ssh-ip_reputation.feed', :feed do
+  let(:provider) { 'arbor' }
+  let(:name) { 'ssh_ip_reputation' }
+  let(:event_types) { [:scanning]}
+
+  it_fetches_url 'http://atlas-public.ec2.arbor.net/public/ssh_attackers'
+
+  describe_parsing_the_file feed_data('arbor_ssh.txt') do
+    it "should have parsed 15 records" do
+      expect(num_records_parsed).to eq(15)
+    end
+    it "should have filtered 1 records" do
+      expect(num_records_filtered).to eq(1)
+    end
+  end
+
+  describe_parsing_a_record '190.255.48.99' do
+    it "should have parsed" do
+      expect(status).to eq(:parsed)
+    end
+    it "should have parsed 1 event" do
+      expect(events.count).to eq(1)
+    end
+	describe 'event 0' do
+      subject { events[0] }
+      its(:type) { is_expected.to be(:scanning) }
+      its(:ipv4s) { is_expected.to  eq(build(:ipv4s, values: ['190.255.48.99'])) }
+    end
+  end
+
+  describe_parsing_a_record '184.172.196.132' do
+    it "should have parsed" do
+      expect(status).to eq(:parsed)
+    end
+    it "should have parsed 1 event" do
+      expect(events.count).to eq(1)
+    end
+	describe 'event 0' do
+      subject { events[0] }
+      its(:type) { is_expected.to be(:scanning) }
+      its(:ipv4s) { is_expected.to  eq(build(:ipv4s, values: ['184.172.196.132'])) }
+    end
+  end
+end

data/spec/feeds/autoshun_shunlist_spec.rb

@@ -0,0 +1,38 @@
+require 'spec_helper'
+
+describe 'feeds/autoshun_shunlist.feed', :feed do
+  let(:provider) { 'autoshun' }
+  let(:name) { 'shunlist' }
+  let(:event_types) { [:scanning]}
+
+  it_fetches_url 'http://www.autoshun.org/files/shunlist.csv'
+
+  describe_parsing_the_file feed_data('autoshun_shunlist.csv') do
+    it "should have parsed 19 records" do
+      expect(num_records_parsed).to eq(19)
+    end
+    it "should have filtered 1 records" do
+      expect(num_records_filtered).to eq(1)
+    end
+  end
+
+  describe_parsing_a_record '1.93.34.230,2014-07-16 08:01:23,SSH Brute Force' do
+    it "should have parsed" do
+      expect(status).to eq(:parsed)
+    end
+    it "should have parsed 1 event" do
+      expect(events.count).to eq(1)
+    end
+    describe 'event 0' do
+      subject { events[0] }
+      its(:type) { is_expected.to be(:scanning) }
+      its(:ipv4s) { is_expected.to  eq(build(:ipv4s, values: ['1.93.34.230'])) }
+    end
+  end
+
+  describe_parsing_a_record 'Shunlist as of Mon, 21 Jul 2014 13:30:02 -0500' do
+    it "should have been filtered" do
+      expect(status).to eq(:filtered)
+    end
+  end
+end

data/spec/feeds/bambenek_c2_masterlist-domain_reputation_spec.rb

@@ -0,0 +1,38 @@
+require 'spec_helper'
+
+describe 'feeds/bambenek_c2_masterlist-domain_reputation.feed', :feed do
+  let(:provider) { 'bambenek' }
+  let(:name) { 'c2_masterlist' }
+  let(:event_types) { [:scanning]}
+
+  it_fetches_url 'http://osint.bambenekconsulting.com/feeds/c2-dommasterlist.txt'
+
+  describe_parsing_the_file feed_data('bambenek_c2-dommasterlist.csv') do
+    it "should have parsed 14 records" do
+      expect(num_records_parsed).to eq(14)
+    end
+    it "should have filtered 16 records" do
+      expect(num_records_filtered).to eq(16)
+    end
+  end
+
+  describe_parsing_a_record 'hhvohslwvpww.ru,Domain used by tinba,2014-11-03 16:30,http://osint.bambenekconsulting.com/manual/tinba-iplist.txt' do
+    it "should have parsed" do
+      expect(status).to eq(:parsed)
+    end
+    it "should have parsed 1 event" do
+      expect(events.count).to eq(1)
+    end
+	describe 'event 0' do
+      subject { events[0] }
+      its(:type) { is_expected.to be(:c2) }
+      its(:fqdns) { is_expected.to match_array(['hhvohslwvpww.ru']) }
+    end
+  end
+
+  describe_parsing_a_record '## jcb@bambenekconsulting.com // http://bambenekconsulting.com' do
+    it "should have been filtered" do
+      expect(status).to eq(:filtered)
+    end
+  end
+end

data/spec/feeds/bambenek_c2_masterlist-ip_reputation_spec.rb

@@ -0,0 +1,39 @@
+require 'spec_helper'
+
+describe 'feeds/bambenek_c2_masterlist-ip_reputation.feed', :feed do
+  let(:provider) { 'bambenek' }
+  let(:name) { 'c2_masterlist_ip' }
+  let(:event_types) { [:scanning]}
+
+
+  it_fetches_url 'http://osint.bambenekconsulting.com/feeds/c2-ipmasterlist.txt'
+
+  describe_parsing_the_file feed_data('bambenek_c2-ipmasterlist.csv') do
+    it "should have parsed 11 records" do
+      expect(num_records_parsed).to eq(11)
+    end
+    it "should have filtered 16 records" do
+      expect(num_records_filtered).to eq(16)
+    end
+  end
+
+  describe_parsing_a_record '212.175.66.70,IP used by matsnu C&C,2014-11-03 16:40,http://osint.bambenekconsulting.com/manual/matsnu-iplist.txt' do
+    it "should have parsed" do
+      expect(status).to eq(:parsed)
+    end
+    it "should have parsed 1 event" do
+      expect(events.count).to eq(1)
+    end
+	describe 'event 0' do
+      subject { events[0] }
+      its(:type) { is_expected.to be(:c2) }
+	  its(:ipv4s) { is_expected.to eq(build(:ipv4s, values: ['212.175.66.70'])) }
+    end
+  end
+
+  describe_parsing_a_record '## jcb@bambenekconsulting.com // http://bambenekconsulting.com' do
+    it "should have been filtered" do
+      expect(status).to eq(:filtered)
+    end
+  end
+end

data/spec/feeds/bambenek_dga_feed-domain_reputation_spec.rb

@@ -0,0 +1,39 @@
+require 'spec_helper'
+
+describe 'feeds/bambenek_dga_feed-domain_reputation.feed', :feed do
+  let(:provider) { 'bambenek' }
+  let(:name) { 'dga_feed' }
+  let(:event_types) { [:scanning]}
+
+
+  it_fetches_url 'http://osint.bambenekconsulting.com/feeds/dga-feed.txt'
+
+  describe_parsing_the_file feed_data('bambenek_dga_feed.csv') do
+    it "should have parsed 28 records" do
+      expect(num_records_parsed).to eq(28)
+    end
+    it "should have filtered 14 records" do
+      expect(num_records_filtered).to eq(14)
+    end
+  end
+
+  describe_parsing_a_record 'rjfifeogqukyjdw.ru,Domain used by Wikipedia 25 DGA for 01 Nov 2014,20141101,http://osint.bambenekconsulting.com/manual/wiki25.txt' do
+    it "should have parsed" do
+      expect(status).to eq(:parsed)
+    end
+    it "should have parsed 1 event" do
+      expect(events.count).to eq(1)
+    end
+	describe 'event 0' do
+      subject { events[0] }
+      its(:type) { is_expected.to be(:c2) }
+      its(:fqdns) { is_expected.to match_array(['rjfifeogqukyjdw.ru']) }
+    end
+  end
+
+  describe_parsing_a_record '## http://osint.bambenekconsulting.com/manual/dga-feed.txt' do
+    it "should have been filtered" do
+      expect(status).to eq(:filtered)
+    end
+  end
+end

data/spec/feeds/berkeley-ip_reputation_spec.rb

@@ -0,0 +1,47 @@
+require 'spec_helper'
+
+describe 'feeds/berkeley-ip_reputation.feed', :feed do
+  let(:provider) { 'berkeley' }
+  let(:name) { 'ip_reputation' }
+  let(:event_types) { [:scanning]}
+
+
+  it_fetches_url 'https://security.berkeley.edu/aggressive_ips/ips'
+
+  describe_parsing_the_file feed_data('berkeley.txt') do
+    it "should have parsed 16 records" do
+      expect(num_records_parsed).to eq(16)
+    end
+    it "should have filtered 13 records" do
+      expect(num_records_filtered).to eq(13)
+    end
+  end
+
+  describe_parsing_a_record 'HOSTILE_IP: 116.10.191.182      LAST_SEEN: 1403615662' do
+    it "should have parsed" do
+      expect(status).to eq(:parsed)
+    end
+    it "should have parsed 1 event" do
+      expect(events.count).to eq(1)
+    end
+	describe 'event 0' do
+      subject { events[0] }
+      its(:type) { is_expected.to be(:scanning) }
+      its(:ipv4s) { is_expected.to  eq(build(:ipv4s, values: ['116.10.191.182'])) }
+    end
+  end
+
+  describe_parsing_a_record 'HOSTILE_IP: 144.0.0.22  LAST_SEEN: 1404389169' do
+    it "should have parsed" do
+      expect(status).to eq(:parsed)
+    end
+    it "should have parsed 1 event" do
+      expect(events.count).to eq(1)
+    end
+	describe 'event 0' do
+      subject { events[0] }
+      its(:type) { is_expected.to be(:scanning) }
+      its(:ipv4s) { is_expected.to  eq(build(:ipv4s, values: ['144.0.0.22'])) }
+    end
+  end
+end

data/spec/feeds/bitcash_cz_blacklist-ip_reputation_spec.rb

@@ -0,0 +1,50 @@
+require 'spec_helper'
+
+describe 'feeds/bitcash_cz_blacklist.feed', :feed do
+  let(:provider) { 'bitcash_cz' }
+  let(:name) { 'blacklist' }
+  let(:event_types) { [:scanning]}
+
+
+  it_fetches_url 'http://bitcash.cz/misc/log/blacklist'
+
+  describe_parsing_the_file feed_data('bitcash_cz_blacklist.txt') do
+    it "should have parsed 3 records" do
+      expect(num_records_parsed).to eq(3)
+    end
+    it "should have filtered 4 records" do
+      expect(num_records_filtered).to eq(4)
+    end
+    it "should have missed 0 records" do
+      expect(num_records_missed).to eq(0)
+    end
+  end
+
+  describe_parsing_a_record '107.22.93.75 #    ec2-107-22-93-75.compute-1.amazonaws.com       last access 2014-07-30 01:45:02' do
+    it "should have parsed" do
+      expect(status).to eq(:parsed)
+    end
+    it "should have parsed 1 event" do
+      expect(events.count).to eq(1)
+    end
+	describe 'event 0' do
+      subject { events[0] }
+      its(:type) { is_expected.to be(:scanning) }
+	  its(:ipv4s) { is_expected.to eq(build(:ipv4s, values: ['107.22.93.75'])) }
+    end
+  end
+
+  describe_parsing_a_record '195.98.179.106 #  broadband-195-98-179-106.2com.net              last access 2014-09-02 17:01:01' do
+    it "should have parsed" do
+      expect(status).to eq(:parsed)
+    end
+    it "should have parsed 1 event" do
+      expect(events.count).to eq(1)
+    end
+	describe 'event 0' do
+      subject { events[0] }
+      its(:type) { is_expected.to be(:scanning) }
+	  its(:ipv4s) { is_expected.to eq(build(:ipv4s, values: ['195.98.179.106'])) }
+    end
+  end
+end

data/spec/feeds/blocklist_de_apache-ip_reputation_spec.rb

@@ -0,0 +1,47 @@
+require 'spec_helper'
+
+describe 'feeds/blocklist_de_apache-ip_reputation.feed', :feed do
+  let(:provider) { 'blocklist_de' }
+  let(:name) { 'apache_ip_reputation' }
+  let(:event_types) { [:scanning]}
+
+
+  it_fetches_url 'http://www.blocklist.de/lists/apache.txt'
+
+  describe_parsing_the_file feed_data('blocklist_de_apache-ip-reputation.txt') do
+    it "should have parsed 15 records" do
+      expect(num_records_parsed).to eq(15)
+    end
+    it "should have filtered 2 records" do
+      expect(num_records_filtered).to eq(2)
+    end
+  end
+
+  describe_parsing_a_record '109.228.235.167' do
+    it "should have parsed" do
+      expect(status).to eq(:parsed)
+    end
+    it "should have parsed 1 event" do
+      expect(events.count).to eq(1)
+    end
+	describe 'event 0' do
+      subject { events[0] }
+      its(:type) { is_expected.to be(:scanning) }
+      its(:ipv4s) { is_expected.to  eq(build(:ipv4s, values: ['109.228.235.167'])) }
+    end
+  end
+
+  describe_parsing_a_record '109.70.54.11' do
+    it "should have parsed" do
+      expect(status).to eq(:parsed)
+    end
+    it "should have parsed 1 event" do
+      expect(events.count).to eq(1)
+    end
+	describe 'event 0' do
+      subject { events[0] }
+      its(:type) { is_expected.to be(:scanning) }
+      its(:ipv4s) { is_expected.to  eq(build(:ipv4s, values: ['109.70.54.11'])) }
+    end
+  end
+end