threatinator 0.1.6 → 0.2.0

data/feeds/vxvault-url_reputation.feed

@@ -0,0 +1,22 @@
+provider "vxvault"
+name "url_reputation"
+fetch_http('http://vxvault.siri-urz.net/URL_List.php')
+
+filter_whitespace
+filter_comments
+
+# Filter out any lines that don't start with a URL
+filter do |record|
+  !(record.data =~ /^http/)
+end
+
+parse_eachline(:separator => "\n") do |event_generator, record|
+  url = record.data.strip
+  next if url.nil?
+
+  event_generator.call() do |event|
+    event.type = :malware_host
+    event.add_url(url) do |url_event|
+	end
+  end
+end

data/feeds/yoyo_adservers-domain_reputation.feed

@@ -0,0 +1,16 @@
+provider "yoyo"
+name "adservers"
+fetch_http('http://pgl.yoyo.org/adservers/serverlist.php?hostformat=nohtml')
+
+filter_whitespace
+filter_comments
+
+parse_eachline(:separator => "\n") do |event_generator, record|
+  domain = record.data.strip()
+  next if domain.nil?
+
+  event_generator.call() do |event|
+    event.type = :scanning
+    event.add_fqdn(domain)
+  end
+end

data/lib/threatinator/actions/run/action.rb

@@ -1,12 +1,16 @@
 require 'threatinator/action'
 require 'threatinator/exceptions'
 require 'threatinator/feed_runner'
+require 'threatinator/logging'
+require 'threatinator/actions/run/status_observer'
 require 'csv'
 
 module Threatinator
   module Actions
     module Run
       class Action < Threatinator::Action
+        include Logging
+
         def initialize(registry, config)
           super(registry)
           @config = config
@@ -21,12 +25,15 @@ module Threatinator
 
           feed = registry.get(@config.feed_provider, @config.feed_name)
           if feed.nil?
+            logger.error("Unknown feed: provider = #{@config.feed_provider}, name = #{@config.feed_name}")
             raise Threatinator::Exceptions::UnknownFeed.new(@config.feed_provider, @config.feed_name)
           end
 
           output = build_output
 
           feed_runner = Threatinator::FeedRunner.new(feed, output)
+          status = StatusObserver.new
+          feed_runner.add_observer(status)
 
           @config.observers.each do |observer|
             feed_runner.add_observer(observer)
@@ -34,10 +41,15 @@ module Threatinator
 
           feed_runner.run
 
-#          if feed_report.num_records_missed != 0
-#            $stderr.puts "WARNING: #{feed_report.num_records_missed} lines/records were MISSED (neither filtered nor parsed). You may need to update your feed specification! Rerun with --coverage to see which records are parsed/filtered/missed" 
-#          end
+          if status.missed?
+            logger.error "#{status.missed} records were MISSED (neither parsed nor filtered). You may need to update your feed specification! Try increasing the logging level to DEBUG, or re-run with run.coverage_output='output.csv' to see which records were parsed/filtered/missed."
+          end
+
+          if status.errors?
+            logger.error "#{status.errors} records had errors! You may have a bug in your feed specification! Try increasing the logging level to DEBUG, or re-run with run.coverage_output='output.csv' to see which records had errors."
+          end
 
+          logger.info "#{status.total} records processed. #{status.parsed} parsed, #{status.filtered} filtered, #{status.missed} missed, #{status.errors} errors"
         end
       end
     end

data/lib/threatinator/actions/run/coverage_observer.rb

@@ -16,18 +16,23 @@ module Threatinator
         end
 
         def open
-          @csv = ::CSV.open(@filename, "wb", :headers => [:status, :event_count, :line_number, :pos_start, :pos_end, :data], :write_headers => true)
+          @csv = ::CSV.open(@filename, "wb", :headers => [:status, :event_count, :line_number, :pos_start, :pos_end, :data, :message], :write_headers => true)
         end
 
         # Handles FeedRunner observations
         def update(message, *args)
           case message
           when :record_missed
-            log_record(:missed, args.shift, [])
+            log_record(:missed, args.shift, 0)
           when :record_filtered
-            log_record(:filtered, args.shift, [])
+            log_record(:filtered, args.shift, 0)
           when :record_parsed
-            log_record(:parsed, args.shift, args.shift)
+            log_record(:parsed, args.shift, args.shift.count)
+          when :record_error
+            record = args.shift
+            errors = args.shift
+            message = errors.map { |e| e.message }.join(', ')
+            log_record(:error, record, 0, message)
           when :end
             close
           when :start
@@ -38,11 +43,11 @@ module Threatinator
         # @param [Symbol] status :parsed, :missed, :filtered
         # @param [Threatinator::Record] record
         # @param [Array<Threatinator::Event>] events
-        def log_record(status, record, events = [])
+        def log_record(status, record, event_count, message = '')
           return if closed?
           @csv.add_row(  [
-            status, events.count, record.line_number, 
-            record.pos_start, record.pos_end, record.data.inspect])
+            status, event_count, record.line_number, 
+            record.pos_start, record.pos_end, record.data.inspect, message])
         end
 
         def close

data/lib/threatinator/actions/run/status_observer.rb

@@ -0,0 +1,37 @@
+module Threatinator
+  module Actions
+    module Run
+      class StatusObserver
+        attr_reader :missed, :filtered, :parsed, :errors
+        def initialize
+          @missed = @filtered = @parsed = @errors = 0
+        end
+
+        def total
+          @missed + @filtered + @parsed + @errors
+        end
+
+        # Handles FeedRunner observations
+        def update(message, *args)
+          case message
+          when :record_missed
+            @missed += 1
+          when :record_filtered
+            @filtered += 1
+          when :record_parsed
+            @parsed += 1
+          when :record_error
+            @errors += 1
+          end
+        end
+
+        def missed?; @missed > 0; end
+        def parsed?; @parsed > 0; end
+        def filtered?; @filtered > 0; end
+        def errors?; @errors > 0; end
+
+      end
+    end
+  end
+end
+

data/lib/threatinator/cli.rb

@@ -1,12 +1,18 @@
 require 'threatinator/cli/parser'
+require 'threatinator/config/logger'
+require 'threatinator/logger'
 module Threatinator
   module CLI
     def self.process!(args)
       parser = Parser.new
       ret = parser.parse(args)
-      if builder = parser.builder
-        builder.build.exec
-      end
+      builder = parser.builder
+      return ret if builder.nil?
+
+      conf = Threatinator::Config::Logger.new(parser.config_hash['logger'])
+      Threatinator::Logger.configure_logger(conf)
+
+      builder.build.exec
       ret
     end
   end

data/lib/threatinator/cli/parser.rb

@@ -2,6 +2,7 @@ require 'gli'
 require 'threatinator/actions/run'
 require 'threatinator/actions/list'
 require 'threatinator/config/feed_search'
+require 'threatinator/config/logger'
 require 'threatinator/cli/list_action_builder'
 require 'threatinator/cli/run_action_builder'
 require 'threatinator/plugin_loader'
@@ -49,7 +50,10 @@ module Threatinator
     end
 
     class Parser
+      include Helpers
+
       attr_accessor :builder
+      attr_reader :config_hash, :extra_args
       attr_reader :run_action_config_class
 
       def initialize
@@ -64,6 +68,11 @@ module Threatinator
         @mod.run(args)
       end
 
+      def set_opts(global_options, options, args)
+        @config_hash = fix_opts(global_options.merge(options))
+        @extra_args = args
+      end
+
       def _init_mod
         parser = self
         Module.new do
@@ -72,6 +81,7 @@ module Threatinator
 
           program_desc 'Threatinator!'
 
+          add_cli_args(self, Threatinator::Config::Logger.properties('logger'))
           add_cli_args(self, Threatinator::Config::FeedSearch.properties('feed_search'))
 
           desc "fetch and parse a feed"
@@ -88,9 +98,9 @@ module Threatinator
                 options["run.feed_name"] = args.shift
               end
 
-              opts = fix_opts(global_options.merge(options))
+              parser.set_opts(global_options, options, args)
 
-              builder = Threatinator::CLI::RunActionBuilder.new(opts, args, parser.run_action_config_class)
+              builder = Threatinator::CLI::RunActionBuilder.new(parser.config_hash, parser.extra_args, parser.run_action_config_class)
               parser.builder = builder
             end
           end
@@ -99,8 +109,8 @@ module Threatinator
           command :list do |c|
             add_cli_args(c, Threatinator::Actions::List::Config.properties('list'))
             c.action do |global_options, options, args|
-              opts = fix_opts(global_options.merge(options))
-              parser.builder = Threatinator::CLI::ListActionBuilder.new(opts, args)
+              parser.set_opts(global_options, options, args)
+              parser.builder = Threatinator::CLI::ListActionBuilder.new(parser.config_hash, parser.extra_args)
             end
           end
         end

data/lib/threatinator/config.rb

@@ -1,6 +1,7 @@
 module Threatinator
   module Config
     require 'threatinator/config/feed_search'
+    require 'threatinator/config/logger'
   end
 end
 

data/lib/threatinator/config/logger.rb

@@ -0,0 +1,14 @@
+require 'threatinator/config/base'
+require 'threatinator/logger'
+
+module Threatinator
+  module Config
+    class Logger < Threatinator::Config::Base
+      attribute :level, String, default: Threatinator::Logger.level_string, 
+        coercer: lambda { |v| v.to_s.upcase },
+        description: "Set the logging level: #{::Threatinator::Logger.levels.join(',')}"
+    end
+  end
+end
+
+

data/lib/threatinator/event.rb

@@ -1,28 +1,38 @@
-require 'threatinator/property_definer'
+require 'threatinator/model/base'
+require 'threatinator/model/observables/ipv4_collection'
+require 'threatinator/model/observables/fqdn_collection'
+require 'threatinator/model/observables/url_collection'
+require 'equalizer'
 require 'set'
 
 module Threatinator
-  class Event
-    include Threatinator::PropertyDefiner
+  class Event < Threatinator::Model::Base
+    include Equalizer.new(:feed_provider, :feed_name, :type, :ipv4s, :fqdns, :urls)
+    attr_reader :feed_provider, :feed_name, :type, :ipv4s, :fqdns, :urls
 
     VALID_TYPES = Set.new([:c2, :attacker, :malware_host, :spamming, :scanning, :phishing])
 
-    def initialize(opts = {})
-      _parse_properties(opts)
-    end
-
-    property :feed_provider, type: String
-    property :feed_name, type: String
-    property :type, type: Symbol, validate: lambda { |obj, val| VALID_TYPES.include?(val) }
-    property :ipv4s, type: Array, default: lambda { Array.new }
-    property :fqdns, type: Array, default: lambda { Array.new }
+    validates :feed_provider, :feed_name, type: ::String
+    validates :type, type: ::Symbol, allow_nil: false, inclusion: {in: VALID_TYPES}
+    validates :ipv4s, type: Threatinator::Model::Observables::Ipv4Collection
+    validates :fqdns, type: Threatinator::Model::Observables::FqdnCollection
+    validates :urls, type: Threatinator::Model::Observables::UrlCollection
 
-    def add_ipv4(ipv4)
-      self.ipv4s << ipv4
-    end
-
-    def add_fqdn(fqdn)
-      self.fqdns << fqdn
+    # @param [Hash] opts
+    # @option opts [String] :feed_provider The name of the feed provider
+    # @option opts [String] :feed_name The name of the feed
+    # @option opts [Symbol] :type The 'type' of feed.
+    # @option opts [#each] :ipv4s A collection of ipv4s
+    # @option opts [#each] :fqdns A collection of FQDNs
+    # @option opts [#each] :urls A collection of Urls
+    def initialize(opts = {})
+      @feed_provider = opts[:feed_provider]
+      @feed_name = opts[:feed_name]
+      @type = opts[:type]
+      @ipv4s = Threatinator::Model::Observables::Ipv4Collection.new(opts[:ipv4s] || [])
+      @fqdns = Threatinator::Model::Observables::FqdnCollection.new(opts[:fqdns] || [])
+      @urls = Threatinator::Model::Observables::UrlCollection.new(opts[:urls] || [])
+      super()
     end
   end
 end

data/lib/threatinator/event_builder.rb

@@ -1,41 +1,70 @@
 require 'threatinator/event'
+require 'threatinator/exceptions'
+require 'threatinator/model/observables/ipv4'
+require 'ip'
 
 module Threatinator
   class EventBuilder
-    attr_reader :total
-    def initialize(feed)
-      @feed = feed
-      @built_events = []
-      @total = 0
+    attr_writer :type
+
+    def initialize(feed_provider, feed_name)
+      @feed_provider = feed_provider
+      @feed_name = feed_name
+      self.reset
     end
 
-    def each_built_event
-      @built_events.each do |event|
-        yield event
-      end
-      @built_events.clear
+    def reset
+      @type = nil
+      @ipv4s = []
+      @fqdns = []
+      @urls = []
     end
 
-    def count
-      @built_events.count
+    def build
+      opts = {
+        feed_provider: @feed_provider,
+        feed_name: @feed_name,
+      }
+      opts[:type] = @type unless @type.nil?
+
+      ret = Threatinator::Event.new(opts)
+
+      @ipv4s.each do |ipv4, opts|
+        opts = opts.dup
+        if ipv4.is_a?(::String)
+          ipv4 = ::IP::V4.parse(ipv4)
+        end
+        opts[:ipv4] = ipv4
+        ret.ipv4s << Threatinator::Model::Observables::Ipv4.new(opts)
+      end
+      @fqdns.each do |fqdn|
+        ret.fqdns << fqdn
+      end
+      @urls.each do |url|
+        url = begin
+          ::Addressable::URI.parse(url)
+        rescue TypeError => e
+          raise Threatinator::Exceptions::EventBuildError, "Failed to parse URL"
+        end
+        ret.urls << url
+      end
+      ret
+    rescue Threatinator::Exceptions::InvalidAttributeError => e
+      raise Threatinator::Exceptions::EventBuildError, e.message
     end
 
-    def clear
-      @built_events.clear
+    def add_fqdn(fqdn)
+      @fqdns << fqdn
     end
 
-    def create_event_proc
-      self.method(:create_event).to_proc
+    def add_ipv4(ipv4, opts = {})
+      @ipv4s << [ipv4, opts]
     end
 
-    def create_event
-      event = Threatinator::Event.new
-      event.feed_provider = @feed.provider
-      event.feed_name = @feed.name
-      yield(event)
-      @total += 1
-      @built_events << event
+    def add_url(url)
+      @urls << url
     end
   end
 end
 
+

data/lib/threatinator/exceptions.rb

@@ -30,12 +30,9 @@ module Threatinator
     end
     
     class InvalidAttributeError < StandardError
-      attr_reader :attribute, :got
-      def initialize(attribute, got)
-        @attribute = attribute
-        @got = got
-        super("Invalid value for attribute '#{attribute}'. Got " + got.inspect)
-      end
+    end
+
+    class EventBuildError < StandardError
     end
 
     class AlreadyRegisteredError < StandardError