threatinator 0.1.1

data/spec/feeds/zeus-ip_reputation_spec.rb

@@ -0,0 +1,50 @@
+require 'spec_helper'
+
+describe 'feeds/zeus-ip_reputation.feed', :feed do
+  let(:provider) { 'abuse_ch' }
+  let(:name) { 'zeus_ip_reputation' }
+
+  it_fetches_url 'https://zeustracker.abuse.ch/blocklist.php?download=ipblocklist'
+
+  describe_parsing_the_file feed_data('zeus-ip_reputation.txt') do
+    it "should have parsed 10 records" do
+      expect(num_records_parsed).to eq(279)
+    end
+    it "should have filtered 8 records" do
+      expect(num_records_filtered).to eq(6)
+    end
+    it "should have missed 0 records" do
+      expect(num_records_missed).to eq(0)
+    end
+  end
+
+  describe_parsing_a_record '109.229.210.250' do
+    it "should have parsed" do
+      expect(status).to eq(:parsed)
+    end
+    it "should have parsed 1 event" do
+      expect(events.count).to eq(1)
+    end
+    describe 'event 0' do
+      subject { events[0] }
+      its(:type) { is_expected.to be(:c2) }
+      its(:ipv4s) { is_expected.to match_array(['109.229.210.250']) }
+    end
+  end
+
+  describe_parsing_a_record '141.105.67.94' do
+    it "should have parsed" do
+      expect(status).to eq(:parsed)
+    end
+    it "should have parsed 1 event" do
+      expect(events.count).to eq(1)
+    end
+    describe 'event 0' do
+      subject { events[0] }
+      its(:type) { is_expected.to be(:c2) }
+      its(:ipv4s) { is_expected.to match_array(['141.105.67.94']) }
+    end
+  end
+end
+
+

data/spec/fixtures/feed/provider1/feed1.feed

@@ -0,0 +1,6 @@
+provider "provider1"
+name "feed1"
+fetch_http('https://foobar/feed1.data')
+
+parse_eachline(:separator => "\n") do |builder, line|
+end

data/spec/fixtures/parsers/test.xml

@@ -0,0 +1,13 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<doc>
+  <a>
+    <b>
+      <c>deep1A</c>
+      <c>deep1B</c>
+    </b>
+    <b>
+      <c>deep2A</c>
+      <c>deep2B</c>
+    </b>
+  </a>
+</doc>

data/spec/fixtures/parsers/test_self_closing.xml

@@ -0,0 +1,20 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<doc>
+  <x>
+    <a />
+    <y>
+      <a />
+      <z foo="bar1"/>
+      <z foo="bar2"/>
+    </y>
+    <y>
+      <a />
+      <a />
+      <z foo="bar3"/>
+      <z foo="bar4"/>
+    </y>
+  </x>
+  <a />
+  <a />
+</doc>
+

data/spec/fixtures/plugins/bad/threatinator/plugins/test_error1/plugin.rb

@@ -0,0 +1 @@
+asldkfj

data/spec/fixtures/plugins/fake.rb

@@ -0,0 +1,19 @@
+require 'threatinator/output'
+module FakeOutputPlugins
+  class Plugin1 < Threatinator::Output
+    class Config < superclass::Config
+      attribute :foo
+    end
+  end
+  class Plugin2 < Threatinator::Output
+    class Config < superclass::Config
+      attribute :bar
+    end
+  end
+  class Plugin3 < Threatinator::Output
+    class Config < superclass::Config
+      attribute :woof
+    end
+  end
+end
+

data/spec/fixtures/plugins/good/threatinator/plugins/test_type1/plugin_a.rb

@@ -0,0 +1,8 @@
+module Threatinator
+  module Plugins
+    module TestType1
+      class PluginA
+      end
+    end
+  end
+end

data/spec/fixtures/plugins/good/threatinator/plugins/test_type1/plugin_b.rb

@@ -0,0 +1,8 @@
+module Threatinator
+  module Plugins
+    module TestType1
+      class PluginB
+      end
+    end
+  end
+end

data/spec/fixtures/plugins/good/threatinator/plugins/test_type2/plugin_c.rb

@@ -0,0 +1,8 @@
+module Threatinator
+  module Plugins
+    module TestType2
+      class PluginC
+      end
+    end
+  end
+end

data/spec/fixtures/plugins/good/threatinator/plugins/test_type2/plugin_d.rb

@@ -0,0 +1,8 @@
+module Threatinator
+  module Plugins
+    module TestType2
+      class PluginD
+      end
+    end
+  end
+end

data/spec/fixtures/plugins/good/threatinator/plugins/test_type3/plugin_e.rb

@@ -0,0 +1,8 @@
+module Threatinator
+  module Plugins
+    module TestType3
+      class PluginE
+      end
+    end
+  end
+end

data/spec/fixtures/plugins/good/threatinator/plugins/test_type3/plugin_f.rb

@@ -0,0 +1,8 @@
+module Threatinator
+  module Plugins
+    module TestType3
+      class PluginF
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,52 @@
+require 'rubygems'
+require 'bundler'
+Bundler.setup :default, :test
+
+require 'pathname'
+SPEC_ROOT = Pathname.new(__FILE__).dirname.expand_path
+PROJECT_ROOT =  SPEC_ROOT.join('../').expand_path
+SUPPORT_ROOT = SPEC_ROOT.join('support')
+
+FEEDS_ROOT =  PROJECT_ROOT.join('feeds')
+FEED_DATA_ROOT =  SPEC_ROOT.join('feeds', 'data')
+
+FIXTURES_ROOT =  SPEC_ROOT.join('fixtures')
+PARSER_FIXTURES = FIXTURES_ROOT.join('parsers')
+PLUGIN_FIXTURES = FIXTURES_ROOT.join('plugins')
+FEED_FIXTURES =   FIXTURES_ROOT.join('feed')
+
+require 'webmock/rspec'
+require 'simplecov'
+require 'rspec/its'
+
+formatters = [
+  SimpleCov::Formatter::HTMLFormatter
+]
+
+begin
+  require 'simplecov-vim/formatter'
+  formatters << SimpleCov::Formatter::VimFormatter
+rescue ::LoadError
+end
+
+SimpleCov.formatters = formatters
+SimpleCov.start do
+  project_root = RSpec::Core::RubyProject.root
+  add_filter PROJECT_ROOT.join('spec').to_s
+  add_filter PROJECT_ROOT.join('.gem').to_s
+  add_filter PROJECT_ROOT.join('.git').to_s
+end 
+
+require 'factory_girl'
+Dir.glob(SUPPORT_ROOT.join('**','*.rb')).sort.each { |f| require f}
+
+RSpec.configure do |config|
+  config.include FactoryGirl::Syntax::Methods
+  config.include IOHelpers
+  config.extend FeedHelpers::FeedHelperMethods, :feed
+
+  config.expect_with :rspec do |c|
+    c.syntax = :expect
+  end
+end
+

data/spec/support/bad_feeds/missing_fetcher.feed

@@ -0,0 +1,7 @@
+provider "provider1"
+name "name1"
+
+parse_eachline(:separator => "\n") do |builder, line|
+end
+
+

data/spec/support/bad_feeds/missing_name.feed

@@ -0,0 +1,6 @@
+provider "provider1"
+fetch_http('https://foobar/feed1.data')
+
+parse_eachline(:separator => "\n") do |builder, line|
+end
+

data/spec/support/bad_feeds/missing_parser.feed

@@ -0,0 +1,3 @@
+provider "provider1"
+name "name1"
+fetch_http('https://foobar/feed1.data')

data/spec/support/bad_feeds/missing_provider.feed

@@ -0,0 +1,5 @@
+name "feed1"
+fetch_http('https://foobar/feed1.data')
+
+parse_eachline(:separator => "\n") do |builder, line|
+end

data/spec/support/factories/event.rb

@@ -0,0 +1,27 @@
+require 'threatinator/event'
+
+FactoryGirl.define do
+  factory :event, class: Threatinator::Event do
+    feed_name 'my_feed_name'
+    feed_provider 'my_provider'
+    type :scanning
+    ipv4s { [ ] }
+    fqdns { [ ] }
+
+    initialize_with { 
+      ret = new() 
+      ret.feed_name = feed_name
+      ret.feed_provider = feed_provider
+      ret.type = type
+      ipv4s.each do |ipv4|
+        ret.add_ipv4(ipv4)
+      end
+      fqdns.each do |fqdn|
+        ret.add_fqdn(fqdn)
+      end
+      ret
+    }
+  end
+end
+
+

data/spec/support/factories/feed.rb

@@ -0,0 +1,32 @@
+require 'threatinator/feed'
+require 'threatinator/parser'
+require 'threatinator/fetcher'
+require 'threatinator/fetchers/http'
+
+FactoryGirl.define do
+  factory :feed, class: Threatinator::Feed do
+    sequence(:provider) { |n| "provider_#{n}" }
+    sequence(:name) { |n| "name_#{n}" }
+    fetcher_builder { lambda { Threatinator::Fetcher.new({}) } } 
+    parser_builder { lambda { Threatinator::Parser.new({}) } } 
+    filter_builders { [] }
+    decoder_builders { [] }
+    parser_block { lambda { |*args| } }
+
+    initialize_with { new(attributes) }
+
+    trait :http do
+      url { "https://foobar/#{provider}/#{name}.data" }
+      fetcher_builder { lambda { Threatinator::Fetchers::Http.new({url: url}) } } 
+    end
+
+    trait :mini do
+      http
+      sequence(:url) { |n| "http://x#{n}" }
+      sequence(:provider) { |n| "x#{n}" }
+      sequence(:name) { |n| "x#{n}" }
+    end
+  end
+
+end
+

data/spec/support/factories/feed_builder.rb

@@ -0,0 +1,65 @@
+require 'threatinator/feed_builder'
+
+FactoryGirl.define do
+  factory :feed_builder, class: Threatinator::FeedBuilder do
+    initialize_with do
+      builder = new()
+      attributes.each_pair do |sym, val|
+        next if val.nil?
+        if val.kind_of?(::Proc)
+          builder.send(sym, &val)
+        else 
+          builder.send(sym, val)
+        end
+      end
+      builder
+    end
+
+    trait :provider do
+      provider 'FakeSecureCo'
+    end
+
+    trait :name do
+      name 'MaliciousDataFeed'
+    end
+
+    trait :http do
+      fetch_http "http://foo.com/bar"
+    end
+
+    trait :parse_eachline do
+      parse_eachline { lambda { |line|  } } 
+    end
+
+    trait :without_provider do
+      name
+      parse_eachline
+      http
+    end
+
+    trait :without_name do
+      provider
+      parse_eachline
+      http
+    end
+
+    trait :without_parser do
+      name
+      provider
+      http
+    end
+
+    trait :without_fetcher do
+      name
+      provider
+      parse_eachline
+    end
+
+    trait :buildable do
+      name
+      provider
+      parse_eachline
+      http
+    end
+  end
+end

data/spec/support/factories/feed_registry.rb

@@ -0,0 +1,8 @@
+require 'threatinator/feed_registry'
+
+FactoryGirl.define do
+  factory :feed_registry, class: Threatinator::FeedRegistry do
+  end
+end
+
+

data/spec/support/factories/output.rb

@@ -0,0 +1,11 @@
+require 'threatinator/output'
+
+FactoryGirl.define do
+  sequence :output_name do |n|
+    name = "output_test#{n}"
+    name.to_sym
+  end
+end
+
+
+

data/spec/support/factories/record.rb

@@ -0,0 +1,17 @@
+require 'threatinator/record'
+
+FactoryGirl.define do
+  factory :record, class: Threatinator::Record do
+    data { "some data" }
+
+    line_number 1
+    pos_start 0
+    pos_end 9
+
+    initialize_with { 
+      new(attributes[:data], attributes) 
+    }
+  end
+end
+
+

data/spec/support/factories/xml_node.rb

@@ -0,0 +1,33 @@
+require 'threatinator/parsers/xml/node'
+
+FactoryGirl.define do
+  factory :xml_node, class: Threatinator::Parsers::XML::Node do
+    name { "foo" }
+    attrs { { } }
+    text { "" }
+    children { [] }
+
+    initialize_with { 
+      new(name, attrs: attrs, text: text, children: children) 
+    }
+
+    trait(:with_attrs) do
+      attrs { {
+        attr1: "val1",
+        attr2: "val2"
+      } }
+    end
+
+    trait(:with_children) do
+      children { [
+        build(:xml_node, name: "child1"),
+        build(:xml_node, name: "child2"),
+        build(:xml_node, name: "child3"),
+      ] }
+    end
+  end
+
+end
+
+
+