shadowbq-threatinator 0.5.0

data/lib/threatinator/feed.rb

@@ -0,0 +1,88 @@
+require 'threatinator/exceptions'
+
+module Threatinator
+  class Feed
+    # @param [Hash] opts Options hash
+    # @option opts [String] :provider The name of the provider
+    # @option opts [String] :name The name of the feed
+    # @option opts [Proc] :parser_block A block that will be called by the
+    #   parser each time it processes a record.
+    # @option opts [Proc] :parser_builder A proc that, when called, will
+    #   return a brand new instance of a Threatinator::Parser.
+    # @option opts [Proc] :fetcher_builder A proc that, when called, will
+    #   return a brand new instance of a Threatinator::Fetcher.
+    # @option opts [Array<Proc>] :filter_builders An array of procs that,
+    #   when called, will each return an instance of a filter (something that
+    #   responds to :filter?)
+    # @option opts [Array<Proc>] :decoder_builders An array of procs that,
+    #   when called, will each return an instance of a Threatinator::Decoder
+    def initialize(opts = {})
+      @provider = opts.delete(:provider)
+      @name = opts.delete(:name)
+      @event_types = opts.delete(:event_types) || [:uknown]
+      @parser_block = opts.delete(:parser_block)
+
+      @parser_builder = opts.delete(:parser_builder)
+      @fetcher_builder = opts.delete(:fetcher_builder)
+      @filter_builders = opts.delete(:filter_builders) || []
+      @decoder_builders = opts.delete(:decoder_builders) || []
+      validate!
+    end
+
+    def provider
+      @provider.dup
+    end
+
+    def name
+      @name.dup
+    end
+
+    def event_types
+      @event_types.dup
+    end
+
+    def parser_block
+      @parser_block
+    end
+
+    def fetcher_builder
+      @fetcher_builder
+    end
+
+    def parser_builder
+      @parser_builder
+    end
+
+    def filter_builders
+      @filter_builders.dup
+    end
+
+    def decoder_builders
+      @decoder_builders.dup
+    end
+
+    def validate!
+      validate_attribute!(:provider, @provider) { |x| x.kind_of?(::String) }
+      validate_attribute!(:name, @name) { |x| x.kind_of?(::String) }
+      validate_attribute!(:event_types, @event_types) { |x| x.kind_of?(::Array) }
+      validate_attribute!(:parser_block, @parser_block) { |x| x.kind_of?(::Proc) }
+      validate_attribute!(:fetcher_builder, @fetcher_builder) { |x| x.kind_of?(::Proc) }
+      validate_attribute!(:parser_builder, @parser_builder) { |x| x.kind_of?(::Proc) }
+      validate_attribute!(:filter_builders, @filter_builders) do |x|
+        x.kind_of?(::Array) &&
+          x.all? { |e| e.kind_of?(::Proc) }
+      end
+
+      validate_attribute!(:decoder_builders, @decoder_builders) do |x|
+        x.kind_of?(::Array) &&
+          x.all? { |e| e.kind_of?(::Proc) }
+      end
+    end
+
+    def validate_attribute!(name, val, &block)
+      unless block.call(val) == true
+        raise Threatinator::Exceptions::InvalidAttributeError.new("Invalid attribute (#{name}). Got: #{val.inspect}")
+      end
+    end
+  end
+end

data/lib/threatinator/feed_builder.rb

@@ -0,0 +1,161 @@
+require 'docile'
+require 'threatinator/feed'
+require 'threatinator/exceptions'
+require 'threatinator/decoders/gzip'
+require 'threatinator/fetchers/http'
+require 'threatinator/parsers/getline'
+require 'threatinator/parsers/csv'
+require 'threatinator/parsers/json'
+require 'threatinator/parsers/xml'
+require 'threatinator/filters/block'
+require 'threatinator/filters/whitespace'
+require 'threatinator/filters/comments'
+
+module Threatinator
+  class FeedBuilder
+    def provider(provider_name)
+      @provider = provider_name
+      self
+    end
+
+    def name(name)
+      @name = name
+      self
+    end
+
+    def event_types(event_types=[:notlabeled])
+      @event_types ||= event_types
+      self
+    end
+
+    def fetch_http(url, opts = {})
+      opts[:url] = url
+      @fetcher_builder = lambda do
+        opts_dup = Marshal.load(Marshal.dump(opts))
+        Threatinator::Fetchers::Http.new(opts_dup)
+      end
+      self
+    end
+
+    def parse_xml(pattern_string, opts = {}, &block)
+      @parser_builder = lambda do
+        pattern = Threatinator::Parsers::XML::Pattern.new(pattern_string)
+        opts_dup = Marshal.load(Marshal.dump(opts))
+        opts_dup[:pattern] = pattern
+        Threatinator::Parsers::XML::Parser.new(opts_dup, &block)
+      end
+      @parser_block = block
+      self
+    end
+
+    def parse_json(opts = {}, &block)
+      @parser_builder = lambda do
+        opts_dup = Marshal.load(Marshal.dump(opts))
+        Threatinator::Parsers::JSON::Parser.new(opts_dup, &block)
+      end
+      @parser_block = block
+      self
+    end
+
+    def parse_eachline(opts = {}, &block)
+      @parser_builder = lambda do
+        opts_dup = Marshal.load(Marshal.dump(opts))
+        Threatinator::Parsers::Getline::Parser.new(opts_dup, &block)
+      end
+      @parser_block = block
+      self
+    end
+
+    def parse_csv(opts = {}, &block)
+      @parser_builder = lambda do
+        opts_dup = Marshal.load(Marshal.dump(opts))
+        Threatinator::Parsers::CSV::Parser.new(opts_dup, &block)
+      end
+      @parser_block = block
+      self
+    end
+
+    # Specify a block filter for the parser
+    def filter(&block)
+      @filter_builders ||= []
+      @filter_builders << lambda { Threatinator::Filters::Block.new(block) }
+      self
+    end
+
+    # Filter out whitespace lines. Only works on line-based text.
+    def filter_whitespace
+      @filter_builders ||= []
+      @filter_builders << lambda { Threatinator::Filters::Whitespace.new }
+      self
+    end
+
+    # Filter out whitespace lines. Only works on line-based text.
+    def filter_comments
+      @filter_builders ||= []
+      @filter_builders << lambda { Threatinator::Filters::Comments.new }
+      self
+    end
+
+    # Add the Gzip decoder
+    def decode_gzip
+      decoder_builders << lambda { Threatinator::Decoders::Gzip.new }
+      self
+    end
+    alias_method :extract_gzip, :decode_gzip
+    alias_method :gunzip, :decode_gzip
+
+    def decoder_builders
+      @decoder_builders ||= []
+    end
+    private :decoder_builders
+
+    def build
+      Feed.new(
+        :provider => @provider,
+        :name => @name,
+        :event_types => @event_types || [:unknown],
+        :parser_block => @parser_block,
+        :fetcher_builder => @fetcher_builder,
+        :parser_builder => @parser_builder,
+        :filter_builders => @filter_builders,
+        :decoder_builders => decoder_builders
+      )
+    end
+
+    # Loads the provided file, and generates a builder from it.
+    # @param [String] filename The name of the file to read the feed from
+    # @raise [FeedFileNotFoundError] if the file is not found
+    def self.from_file(filename)
+      begin
+        filedata = File.read(filename)
+      rescue Errno::ENOENT
+        raise Threatinator::Exceptions::FeedFileNotFoundError.new(filename)
+      end
+      from_string(filedata, filename, 0)
+    end
+
+    # Generates a builder from a string via eval.
+    # @param [String] str The DSL code that specifies the feed.
+    # @param [String] filename (nil) Passed to eval.
+    # @param [String] lineno (nil) Passed to eval.
+    # @raise [FeedFileNotFoundError] if the file is not found
+    # @see Kernel#eval for details on filename and lineno
+    def self.from_string(str, filename = nil, lineno = nil)
+      from_dsl do
+        args = [str, binding]
+        unless filename.nil?
+          args << filename
+          unless lineno.nil?
+            args << lineno
+          end
+        end
+        eval(*args)
+      end
+    end
+
+    # Executes the block parameter within DSL scope
+    def self.from_dsl(&block)
+      Docile.dsl_eval(self.new, &block)
+    end
+  end
+end

data/lib/threatinator/feed_registry.rb

@@ -0,0 +1,47 @@
+require 'threatinator/registry'
+require 'threatinator/feed_builder'
+
+module Threatinator
+  class FeedRegistry < Registry
+    # @param [Threatinator::Feed] feed The feed to register
+    # @raise [Threatinator::Exceptions::AlreadyRegisteredError] if a feed
+    #  with the same name and provider is already registered.
+    def register(feed)
+      super([feed.provider, feed.name], feed)
+    end
+
+    # @param [String] provider
+    # @param [String] name
+    # @return [Threatinator::Feed]
+    def get(provider, name)
+      super([provider, name])
+    end
+
+    def each
+      return enum_for(:each) unless block_given?
+      super do |key, feed|
+        yield(feed)
+      end
+    end
+
+    def register_from_file(filename)
+      builder = Threatinator::FeedBuilder.from_file(filename)
+      feed = builder.build
+      register(feed)
+      feed
+    end
+
+    # Builds a new FeedRegistry based on the provided config
+    # @param [Threatinator::Config::FeedSearch] config The configuration
+    def self.build(config)
+      ret = self.new
+      config.search_path.each do |path|
+        pattern = File.join(path, "**", "*.feed")
+        Dir.glob(pattern).each do |filename|
+          ret.register_from_file(filename)
+        end
+      end
+      ret
+    end
+  end
+end

data/lib/threatinator/feed_runner.rb

@@ -0,0 +1,177 @@
+require 'threatinator/event_builder'
+require 'threatinator/logging'
+require 'observer'
+
+module Threatinator
+  # Runs those feeds!
+  #
+  #  Has the following observations:
+  #    :start                 - start of feed parsing
+  #
+  #    :start_fetch           - start of fetching
+  #    :end_fetch             - end of fetching
+  #
+  #    :start_decode          - start of decoding
+  #    :end_decode            - end of decoding
+  #
+  #    :start_parse_record    - start of record parse
+  #       - record                - The record
+  #
+  #    :record_filtered       - Indicates that the record was filtered.
+  #       - record                - The record
+  #
+  #    :record_missed         - Indicates that the record was not parsed
+  #       - record                - The record
+  #
+  #    :record_parsed         - Indicates that the record WAS parsed
+  #       - record                - The record
+  #       - events                - The events that were parsed out of the 
+  #                                 record
+  #
+  #    :record_error         - Indicates that the record WAS parsed
+  #       - record                - The record
+  #       - errors           - An array of exceptions that caused the error
+  #
+  #    :end_parse_record      - when a record has been parsed
+  #       - record                - The record
+  #
+  #    :end                   - completion of feed parsing
+  #
+  class FeedRunner
+    include Observable
+    include Logging
+
+    # @param [Threatinator::Feed] feed The feed that we want to run.
+    # @param [Threatinator::Output] output_formatter
+    def initialize(feed, output_formatter, opts = {})
+      @feed = feed
+      @output_formatter = output_formatter
+      @feed_filters = @feed.filter_builders.map { |x| x.call } 
+      @decoders = @feed.decoder_builders.map { |x| x.call } 
+      @parser_block = @feed.parser_block
+      @create_event_proc = self.method(:create_event).to_proc
+
+      @event_builder = Threatinator::EventBuilder.new(@feed.provider, @feed.name)
+
+      @total_events_built = 0
+      @event_errors = []
+      @built_events = []
+    end
+
+    # @param [Hash] opts The options hash
+    # @option opts [IO-like] :io Override the fetcher by providing 
+    #  an IO directly. 
+    # @option opts [Boolean] :skip_decoding (false) Skip all decoding if set 
+    #  to true. Useful for testing.
+    def run(opts = {})
+      ios = [ ]
+      logger.debug("#run starting #{@feed.provider}:#{@feed.name}") if logger.debug?
+      start = Time.now
+      changed(true); notify_observers(:start)
+      skip_decoding = !!opts.delete(:skip_decoding)
+
+
+      unless io = opts.delete(:io)
+        fetcher = @feed.fetcher_builder.call()
+        changed(true); notify_observers(:start_fetch)
+        io = fetcher.fetch()
+        changed(true); notify_observers(:end_fetch)
+      else
+        logger.debug('#run Skipping fetch. IO object was provided')
+      end
+
+      ios << io
+
+      unless skip_decoding == true
+        changed(true); notify_observers(:start_decode)
+        @decoders.each do |decoder|
+          new_io = decoder.decode(io)
+          ios << new_io
+          io = new_io
+        end
+        changed(true); notify_observers(:end_decode)
+      end
+
+      parser = @feed.parser_builder.call()
+
+      parser.run(io) do |record|
+        rr = parse_record(record)
+      end
+
+      changed(true); notify_observers(:end)
+      
+      logger.debug("#run finished #{@feed.provider}:#{@feed.name} in #{Time.now - start} seconds") if logger.debug?
+      nil
+    ensure
+      # Close all IO objects that we've seen. 
+      while some_io = ios.pop
+        unless some_io.closed?
+          begin
+            some_io.close
+          rescue => e
+            #:nocov:
+            logger.warn("Failed to close IO: #{e} #{e.message}")
+            #:nocov:
+          end
+        end
+      end
+
+      @output_formatter.finish
+    end
+
+    def create_event
+      @event_builder.reset
+      @event_errors.clear
+      yield(@event_builder)
+      begin
+        event = @event_builder.build
+        @total_events_built += 1
+        @built_events << event
+      rescue Threatinator::Exceptions::EventBuildError => e
+        @event_errors << e
+      end
+    end
+
+    def parse_record(record)
+      @built_events.clear
+      events = []
+      changed(true); notify_observers(:start_parse_record, record)
+
+      if @feed_filters.any? { |filter| filter.filter?(record) }
+        changed(true); notify_observers(:record_filtered, record)
+        return
+      end
+
+      @parser_block.call(@create_event_proc, record)
+
+      if @event_errors.count > 0
+        changed(true); notify_observers(:record_error, record, @event_errors)
+        position = "line: #{record.line_number}, start: #{record.pos_start}, end: #{record.pos_end}"
+        messages = @event_errors.map { |e| e.to_s }.join(', ')
+        logger.debug("Error generating event from record (#{position}): #{messages}")
+      elsif @built_events.count == 0
+        changed(true); notify_observers(:record_missed, record)
+        position = "line: #{record.line_number}, start: #{record.pos_start}, end: #{record.pos_end}"
+        logger.debug("Expected event to be generated, but got none from record (#{position})")
+      else 
+        @built_events.each do |event|
+          events << event
+          @output_formatter.handle_event(event)
+        end
+        changed(true); notify_observers(:record_parsed, record, events)
+      end
+      return
+    ensure 
+      changed(true); notify_observers(:end_parse_record, record)
+    end
+
+    # Runs a feed
+    # @param [Threatinator::Feed] feed The feed to run
+    # @param [Threatinator::Output] output The output instance
+    # @param [Hash] run_opts Options passed to #run. See #run .
+    def self.run(feed, output, run_opts = {})
+      self.new(feed, output).run(run_opts)
+    end
+
+  end
+end

data/lib/threatinator/fetcher.rb

@@ -0,0 +1,22 @@
+module Threatinator
+  class Fetcher
+
+    # @param [Hash] opts An options hash. See subclasses for details.
+    def initialize(opts = {})
+    end
+
+    # @return [IO] an IO object
+    def fetch
+      raise NotImplementedError.new("#{self.class}#fetch not implemented!")
+    end
+
+    def ==(other)
+      true
+    end
+
+    def eql?(other)
+      self.class == other.class &&
+        self == other
+    end
+  end
+end

data/lib/threatinator/fetchers/http.rb

@@ -0,0 +1,50 @@
+require 'threatinator/fetcher'
+require 'threatinator/exceptions'
+require 'typhoeus'
+require 'tempfile'
+
+module Threatinator
+  module Fetchers
+    class Http < Threatinator::Fetcher
+      attr_reader :url
+      # @param [Hash] opts An options hash.
+      # @option opts [Addressable::URI] :url The URL that is to be fetched
+      #   (required)
+      #
+      def initialize(opts = {})
+        @url = opts.delete(:url) or raise ArgumentError.new("Missing :url")
+        super(opts)
+      end
+
+      def ==(other)
+        @url == other.url && super(other)
+      end
+
+      # @return [IO] an IO-style object.
+      # @raise [Threatinator::Exceptions::FetchFailed] if the fetch fails
+      def fetch
+        tempfile = Tempfile.new("threatinator_http")
+        request = Typhoeus::Request.new(@url,
+                                        ssl_verifypeer: false,
+                                        followlocation: true,
+                                        forbid_reuse: true
+                                       )
+        request.on_headers do |response|
+          if response.response_code != 200
+
+            raise Threatinator::Exceptions::FetchFailed.new("Request failed!")
+          end
+        end
+        request.on_body do |chunk|
+          tempfile.write(chunk)
+        end
+        # Run it
+        request.run
+        # Reset the IO to the beginning of the file
+        tempfile.rewind
+        tempfile
+      end
+    end
+
+  end
+end

data/lib/threatinator/filter.rb

@@ -0,0 +1,12 @@
+module Threatinator
+  # Acts as a filter for parser data.
+  class Filter
+    # What is passed in as arguments depends upon the parser. 
+    #
+    # @param [Threatinator::Record] record The record to filter
+    # @return [Boolean] true if filtered, false otherwise.
+    def filter?(record)
+      raise NotImplementedError.new("#{self.class}.filter?(record) not implemented")
+    end
+  end
+end

data/lib/threatinator/filters/block.rb

@@ -0,0 +1,18 @@
+require 'threatinator/filter'
+
+module Threatinator
+  module Filters
+    # Basic filter that allows for arbitrary filtering.
+    class Block < Threatinator::Filter
+      def initialize(block)
+        @block = block
+      end
+
+      # @param [Threatinator::Record] record The record to filter
+      # @return [Boolean] true if filtered, false otherwise.
+      def filter?(record)
+        !! @block.call(record)
+      end
+    end
+  end
+end

data/lib/threatinator/filters/comments.rb

@@ -0,0 +1,16 @@
+require 'threatinator/filter'
+
+module Threatinator
+  module Filters
+    # Filters out any lines of text that begin with a comment '#'
+    class Comments < Threatinator::Filter
+      # @param [Threatinator::Record] record The record to filter
+      # @return [Boolean] true if filtered, false otherwise.
+      def filter?(record)
+        record.data[0] == '#'
+      end
+    end
+  end
+end
+
+

data/lib/threatinator/filters/whitespace.rb

@@ -0,0 +1,19 @@
+require 'threatinator/filter'
+
+module Threatinator
+  module Filters
+    # Filters on any lines of text that consist entirely of whitespace
+    class Whitespace < Threatinator::Filter
+      def initialize()
+        @re = /^\s*$/
+      end
+
+      # @param [Threatinator::Record] record The record to filter
+      # @return [Boolean] true if filtered, false otherwise.
+      def filter?(record)
+        !! @re.match(record.data)
+      end
+    end
+  end
+end
+

data/lib/threatinator/logger.rb

@@ -0,0 +1,66 @@
+require 'log4r'
+
+module Threatinator
+  module Logger
+    # The log levels don't get defined until the first time a logger is 
+    # initialized. So, we call the root logger once just to get this to happen.
+    Log4r::RootLogger.instance
+
+    def self.logger_for(name)
+      ::Log4r::Logger[name] || ::Log4r::Logger.new(name)
+    end
+
+    def self.default_logger
+      return @logger unless @logger.nil?
+
+      @logger = logger_for('Threatinator')
+      formatter = ::Log4r::PatternFormatter.new(:pattern => '[%d] %l %C: %M')
+
+      console_outputter = ::Log4r::StderrOutputter.new('console', formatter: formatter)
+      @logger.add console_outputter
+
+      @logger.level = ::Log4r::INFO
+      @logger
+    end
+
+    # @param [Threatinator::Config::Logger] config Logging configuration object
+    def self.configure_logger(config)
+      if config.level
+        if l = self.levels.index(config.level)
+          default_logger.level = l
+        else
+          default_logger.warn("Ignoring unknown logging level: #{config.level.inspect}.")
+        end
+      end
+    end
+
+    def self.level
+      default_logger.level
+    end
+
+    def self.level=(l)
+      default_logger.level = l
+    end
+
+    def self.level_string
+      levels[level]
+    end
+
+    def self.levels
+      default_logger.levels
+    end
+
+    # Initializes the default logger. This allows us to pull the logger levels.
+    self.default_logger()
+
+    module Levels
+      ALL   = ::Log4r::ALL
+      DEBUG = ::Log4r::DEBUG
+      INFO  = ::Log4r::INFO
+      WARN  = ::Log4r::WARN
+      ERROR = ::Log4r::ERROR
+      FATAL = ::Log4r::FATAL
+      OFF   = ::Log4r::OFF
+    end
+  end
+end

data/lib/threatinator/logging.rb

@@ -0,0 +1,20 @@
+require 'threatinator/logger'
+
+module Threatinator
+  # Mixin for mixing logging facilities into classes. 
+  module Logging
+    def self.included(base)
+      base.extend(LoggingClassMethods)
+    end
+
+    def logger
+      @logger ||= self.class.logger
+    end
+  end
+
+  module LoggingClassMethods
+    def logger
+      @logger ||= Threatinator::Logger.logger_for(self.name)
+    end
+  end
+end