threatinator 0.1.6 → 0.2.0

data/lib/threatinator/feed.rb

@@ -75,7 +75,7 @@ module Threatinator
 
     def validate_attribute!(name, val, &block)
       unless block.call(val) == true
-        raise Threatinator::Exceptions::InvalidAttributeError.new(name, val)
+        raise Threatinator::Exceptions::InvalidAttributeError.new("Invalid attribute (#{name}). Got: #{val.inspect}")
       end
     end
   end

data/lib/threatinator/feed_runner.rb

@@ -1,4 +1,5 @@
 require 'threatinator/event_builder'
+require 'threatinator/logging'
 require 'observer'
 
 module Threatinator
@@ -27,6 +28,10 @@ module Threatinator
   #       - events                - The events that were parsed out of the 
   #                                 record
   #
+  #    :record_error         - Indicates that the record WAS parsed
+  #       - record                - The record
+  #       - errors           - An array of exceptions that caused the error
+  #
   #    :end_parse_record      - when a record has been parsed
   #       - record                - The record
   #
@@ -34,17 +39,23 @@ module Threatinator
   #
   class FeedRunner
     include Observable
+    include Logging
 
     # @param [Threatinator::Feed] feed The feed that we want to run.
     # @param [Threatinator::Output] output_formatter
     def initialize(feed, output_formatter, opts = {})
       @feed = feed
       @output_formatter = output_formatter
-      @event_builder = Threatinator::EventBuilder.new(@feed)
       @feed_filters = @feed.filter_builders.map { |x| x.call } 
       @decoders = @feed.decoder_builders.map { |x| x.call } 
       @parser_block = @feed.parser_block
-      @create_event_proc = @event_builder.create_event_proc()
+      @create_event_proc = self.method(:create_event).to_proc
+
+      @event_builder = Threatinator::EventBuilder.new(@feed.provider, @feed.name)
+
+      @total_events_built = 0
+      @event_errors = []
+      @built_events = []
     end
 
     # @param [Hash] opts The options hash
@@ -53,20 +64,30 @@ module Threatinator
     # @option opts [Boolean] :skip_decoding (false) Skip all decoding if set 
     #  to true. Useful for testing.
     def run(opts = {})
+      ios = [ ]
+      logger.debug("#run starting #{@feed.provider}:#{@feed.name}") if logger.debug?
+      start = Time.now
       changed(true); notify_observers(:start)
       skip_decoding = !!opts.delete(:skip_decoding)
 
+
       unless io = opts.delete(:io)
         fetcher = @feed.fetcher_builder.call()
         changed(true); notify_observers(:start_fetch)
         io = fetcher.fetch()
         changed(true); notify_observers(:end_fetch)
+      else
+        logger.debug('#run Skipping fetch. IO object was provided')
       end
 
+      ios << io
+
       unless skip_decoding == true
         changed(true); notify_observers(:start_decode)
         @decoders.each do |decoder|
-          io = decoder.decode(io)
+          new_io = decoder.decode(io)
+          ios << new_io
+          io = new_io
         end
         changed(true); notify_observers(:end_decode)
       end
@@ -78,11 +99,38 @@ module Threatinator
       end
 
       changed(true); notify_observers(:end)
+      
+      logger.debug("#run finished #{@feed.provider}:#{@feed.name} in #{Time.now - start} seconds") if logger.debug?
       nil
+    ensure
+      # Close all IO objects that we've seen. 
+      while some_io = ios.pop
+        unless some_io.closed?
+          begin
+            some_io.close
+          rescue => e
+            logger.warn("Failed to close IO: #{e} #{e.message}")
+          end
+        end
+      end
+
+    end
+
+    def create_event
+      @event_builder.reset
+      @event_errors.clear
+      yield(@event_builder)
+      begin
+        event = @event_builder.build
+        @total_events_built += 1
+        @built_events << event
+      rescue Threatinator::Exceptions::EventBuildError => e
+        @event_errors << e
+      end
     end
 
     def parse_record(record)
-      @event_builder.clear
+      @built_events.clear
       events = []
       changed(true); notify_observers(:start_parse_record, record)
 
@@ -90,12 +138,20 @@ module Threatinator
         changed(true); notify_observers(:record_filtered, record)
         return
       end
+
       @parser_block.call(@create_event_proc, record)
-      if @event_builder.count == 0
+
+      if @event_errors.count > 0
+        changed(true); notify_observers(:record_error, record, @event_errors)
+        position = "line: #{record.line_number}, start: #{record.pos_start}, end: #{record.pos_end}"
+        messages = @event_errors.map { |e| e.to_s }.join(', ')
+        logger.debug("Error generating event from record (#{position}): #{messages}")
+      elsif @built_events.count == 0
         changed(true); notify_observers(:record_missed, record)
-        # Keep track of the fact that this line did not generate any events?
+        position = "line: #{record.line_number}, start: #{record.pos_start}, end: #{record.pos_end}"
+        logger.debug("Expected event to be generated, but got none from record (#{position})")
       else 
-        @event_builder.each_built_event do |event|
+        @built_events.each do |event|
           events << event
           @output_formatter.handle_event(event)
         end

data/lib/threatinator/logger.rb

@@ -0,0 +1,66 @@
+require 'log4r'
+
+module Threatinator
+  module Logger
+    # The log levels don't get defined until the first time a logger is 
+    # initialized. So, we call the root logger once just to get this to happen.
+    Log4r::RootLogger.instance
+
+    def self.logger_for(name)
+      ::Log4r::Logger[name] || ::Log4r::Logger.new(name)
+    end
+
+    def self.default_logger
+      return @logger unless @logger.nil?
+
+      @logger = logger_for('Threatinator')
+      formatter = ::Log4r::PatternFormatter.new(:pattern => '[%d] %l %C: %M')
+
+      console_outputter = ::Log4r::StderrOutputter.new('console', formatter: formatter)
+      @logger.add console_outputter
+
+      @logger.level = ::Log4r::INFO
+      @logger
+    end
+
+    # @param [Threatinator::Config::Logger] config Logging configuration object
+    def self.configure_logger(config)
+      if config.level
+        if l = self.levels.index(config.level)
+          default_logger.level = l
+        else
+          default_logger.warn("Ignoring unknown logging level: #{config.level.inspect}.")
+        end
+      end
+    end
+
+    def self.level
+      default_logger.level
+    end
+
+    def self.level=(l)
+      default_logger.level = l
+    end
+
+    def self.level_string
+      levels[level]
+    end
+
+    def self.levels
+      default_logger.levels
+    end
+
+    # Initializes the default logger. This allows us to pull the logger levels.
+    self.default_logger()
+
+    module Levels
+      ALL   = ::Log4r::ALL
+      DEBUG = ::Log4r::DEBUG
+      INFO  = ::Log4r::INFO
+      WARN  = ::Log4r::WARN
+      ERROR = ::Log4r::ERROR
+      FATAL = ::Log4r::FATAL
+      OFF   = ::Log4r::OFF
+    end
+  end
+end

data/lib/threatinator/logging.rb

@@ -0,0 +1,20 @@
+require 'threatinator/logger'
+
+module Threatinator
+  # Mixin for mixing logging facilities into classes. 
+  module Logging
+    def self.included(base)
+      base.extend(LoggingClassMethods)
+    end
+
+    def logger
+      @logger ||= self.class.logger
+    end
+  end
+
+  module LoggingClassMethods
+    def logger
+      @logger ||= Threatinator::Logger.logger_for(self.name)
+    end
+  end
+end

data/lib/threatinator/model/base.rb

@@ -0,0 +1,23 @@
+require 'active_model'
+require 'active_model/validations'
+require 'threatinator/model/validations'
+require 'threatinator/exceptions'
+
+module Threatinator
+  module Model
+    class Base
+      include ActiveModel::Validations
+      include Threatinator::Model::Validations
+
+      def initialize
+        validate!
+      end
+
+      def validate!
+        unless valid?
+          raise Threatinator::Exceptions::InvalidAttributeError, errors.full_messages.join("\n")
+        end
+      end
+    end
+  end
+end

data/lib/threatinator/model/collection.rb

@@ -0,0 +1,64 @@
+require 'threatinator/exceptions'
+require 'set'
+
+module Threatinator
+  module Model
+    class Collection
+      def initialize(values = [])
+        @collection = Set.new
+        values.each do |v|
+          self << v
+        end
+      end
+
+      def valid_member?(v)
+        #:nocov:
+        raise NotImplementedError, "#valid_member? not implemented"
+        #:nocov:
+      end
+
+      def <<(v)
+        unless valid_member?(v)
+          raise Threatinator::Exceptions::InvalidAttributeError, "Invalid member: #{v.class} '#{v.inspect}'"
+        end
+        @collection << v
+      end
+
+      def include?(member)
+        @collection.include?(member)
+      end
+
+      # @return [Boolean] true if empty, false otherwise
+      def empty?
+        @collection.empty?
+      end
+
+      # @return [Integer] the number of members in the collection
+      def count
+        @collection.count
+      end
+
+      def to_ary
+        @collection.to_a
+      end
+      alias_method :to_a, :to_ary
+
+      def each
+        return to_enum(:each) unless block_given?
+        @collection.each { |v| yield v }
+      end
+
+      def ==(other)
+        if self.equal?(other)
+          return true
+        elsif other.instance_of?(self.class)
+          @collection == other.instance_variable_get(:@collection)
+        else
+          false
+        end
+      end
+
+    end
+
+  end
+end

data/lib/threatinator/model/observables/fqdn_collection.rb

@@ -0,0 +1,13 @@
+require 'threatinator/model/collection'
+
+module Threatinator
+  module Model
+    module Observables
+      class FqdnCollection < Threatinator::Model::Collection
+        def valid_member?(v)
+          v.is_a?(::String)
+        end
+      end
+    end
+  end
+end

data/lib/threatinator/model/observables/ipv4.rb

@@ -0,0 +1,30 @@
+require 'threatinator/model/base'
+require 'ip'
+require 'equalizer'
+
+module Threatinator
+  module Model
+    module Observables
+      class Ipv4 < Threatinator::Model::Base
+        include Equalizer.new(:ipv4)
+        attr_reader :ipv4
+
+        validates_each :ipv4 do |record, attr, value|
+          if value.is_a?(::IP::V4)
+            record.errors.add(attr, 'prefix length is not 32 bits') unless value.pfxlen == 32
+          else
+            record.errors.add(attr, 'not an IP::V4 object')
+          end
+        end
+
+        # @param [Hash] opts
+        # @option opts [IP::V4] :ipv4 An ipv4 object
+        def initialize(opts = {})
+          @ipv4 = opts.delete(:ipv4)
+          super()
+        end
+      end
+    end
+  end
+end
+

data/lib/threatinator/model/observables/ipv4_collection.rb

@@ -0,0 +1,14 @@
+require 'threatinator/model/collection'
+require 'threatinator/model/observables/ipv4'
+
+module Threatinator
+  module Model
+    module Observables
+      class Ipv4Collection < Threatinator::Model::Collection
+        def valid_member?(v)
+          v.kind_of?(Threatinator::Model::Observables::Ipv4)
+        end
+      end
+    end
+  end
+end

data/lib/threatinator/model/observables/url_collection.rb

@@ -0,0 +1,16 @@
+require 'threatinator/model/collection'
+require 'addressable/uri'
+
+module Threatinator
+  module Model
+    module Observables
+      class UrlCollection < Threatinator::Model::Collection
+        def valid_member?(v)
+          v.is_a?(::Addressable::URI) &&
+            v.absolute?
+        end
+      end
+    end
+  end
+end
+

data/lib/threatinator/model/validations.rb

@@ -0,0 +1 @@
+require 'threatinator/model/validations/type'

data/lib/threatinator/model/validations/type.rb

@@ -0,0 +1,21 @@
+require 'active_model'
+require 'active_model/validations'
+
+module Threatinator
+  module Model
+    module Validations
+      class TypeValidator < ActiveModel::EachValidator
+        def initialize(options)
+          @type = options.delete(:with)
+          super
+        end
+
+        def validate_each(record, name, value)
+          unless value.is_a?(@type)
+            record.errors.add name, "Expected to be #{@type}, got #{value.class} #{value.inspect}"
+          end
+        end
+      end
+    end
+  end
+end 

data/lib/threatinator/plugins/output/csv.rb

@@ -22,23 +22,34 @@ module Threatinator
                              :fqdn_1,
                              :fqdn_2,
                              :fqdn_3,
-                             :fqdn_4
+                             :fqdn_4,
+                             :url_1,
+                             :url_2,
+                             :url_3,
+                             :url_4
                            ])
         end
 
         def handle_event(event)
+          ipv4s = event.ipv4s.to_a[0..3].map { |o| o.nil? ? nil : o.ipv4.to_addr }
+          fqdns = event.fqdns.to_a[0..3]
+          urls  = event.urls.to_a[0..3].map {|x| x.to_s }
           @csv.add_row([
             event.feed_provider,
             event.feed_name,
             event.type,
-            event.ipv4s[0],
-            event.ipv4s[1],
-            event.ipv4s[2],
-            event.ipv4s[3],
-            event.fqdns[0],
-            event.fqdns[1],
-            event.fqdns[2],
-            event.fqdns[3]
+            ipv4s[0],
+            ipv4s[1],
+            ipv4s[2],
+            ipv4s[3],
+            fqdns[0],
+            fqdns[1],
+            fqdns[2],
+            fqdns[3],
+            urls[0],
+            urls[1],
+            urls[2],
+            urls[3],
           ])
         end
       end