threatinator 0.1.1

data/lib/threatinator/feed_registry.rb

@@ -0,0 +1,47 @@
+require 'threatinator/registry'
+require 'threatinator/feed_builder'
+
+module Threatinator
+  class FeedRegistry < Registry
+    # @param [Threatinator::Feed] feed The feed to register
+    # @raise [Threatinator::Exceptions::AlreadyRegisteredError] if a feed
+    #  with the same name and provider is already registered.
+    def register(feed)
+      super([feed.provider, feed.name], feed)
+    end
+
+    # @param [String] provider
+    # @param [String] name
+    # @return [Threatinator::Feed]
+    def get(provider, name)
+      super([provider, name])
+    end
+
+    def each
+      return enum_for(:each) unless block_given?
+      super do |key, feed|
+        yield(feed)
+      end
+    end
+
+    def register_from_file(filename)
+      builder = Threatinator::FeedBuilder.from_file(filename)
+      feed = builder.build
+      register(feed)
+      feed
+    end
+
+    # Builds a new FeedRegistry based on the provided config
+    # @param [Threatinator::Config::FeedSearch] config The configuration
+    def self.build(config)
+      ret = self.new
+      config.search_path.each do |path|
+        pattern = File.join(path, "**", "*.feed")
+        Dir.glob(pattern).each do |filename|
+          ret.register_from_file(filename)
+        end
+      end
+      ret
+    end
+  end
+end

data/lib/threatinator/feed_runner.rb

@@ -0,0 +1,118 @@
+require 'threatinator/event_builder'
+require 'observer'
+
+module Threatinator
+  # Runs those feeds!
+  #
+  #  Has the following observations:
+  #    :start                 - start of feed parsing
+  #
+  #    :start_fetch           - start of fetching
+  #    :end_fetch             - end of fetching
+  #
+  #    :start_decode          - start of decoding
+  #    :end_decode            - end of decoding
+  #
+  #    :start_parse_record    - start of record parse
+  #       - record                - The record
+  #
+  #    :record_filtered       - Indicates that the record was filtered.
+  #       - record                - The record
+  #
+  #    :record_missed         - Indicates that the record was not parsed
+  #       - record                - The record
+  #
+  #    :record_parsed         - Indicates that the record WAS parsed
+  #       - record                - The record
+  #       - events                - The events that were parsed out of the 
+  #                                 record
+  #
+  #    :end_parse_record      - when a record has been parsed
+  #       - record                - The record
+  #
+  #    :end                   - completion of feed parsing
+  #
+  class FeedRunner
+    include Observable
+
+    # @param [Threatinator::Feed] feed The feed that we want to run.
+    # @param [Threatinator::Output] output_formatter
+    def initialize(feed, output_formatter, opts = {})
+      @feed = feed
+      @output_formatter = output_formatter
+      @event_builder = Threatinator::EventBuilder.new(@feed)
+      @feed_filters = @feed.filter_builders.map { |x| x.call } 
+      @decoders = @feed.decoder_builders.map { |x| x.call } 
+      @parser_block = @feed.parser_block
+      @create_event_proc = @event_builder.create_event_proc()
+    end
+
+    # @param [Hash] opts The options hash
+    # @option opts [IO-like] :io Override the fetcher by providing 
+    #  an IO directly. 
+    # @option opts [Boolean] :skip_decoding (false) Skip all decoding if set 
+    #  to true. Useful for testing.
+    def run(opts = {})
+      changed(true); notify_observers(:start)
+      skip_decoding = !!opts.delete(:skip_decoding)
+
+      unless io = opts.delete(:io)
+        fetcher = @feed.fetcher_builder.call()
+        changed(true); notify_observers(:start_fetch)
+        io = fetcher.fetch()
+        changed(true); notify_observers(:end_fetch)
+      end
+
+      unless skip_decoding == true
+        changed(true); notify_observers(:start_decode)
+        @decoders.each do |decoder|
+          io = decoder.decode(io)
+        end
+        changed(true); notify_observers(:end_decode)
+      end
+
+      parser = @feed.parser_builder.call()
+
+      parser.run(io) do |record|
+        rr = parse_record(record)
+      end
+
+      changed(true); notify_observers(:end)
+      nil
+    end
+
+    def parse_record(record)
+      @event_builder.clear
+      events = []
+      changed(true); notify_observers(:start_parse_record, record)
+
+      if @feed_filters.any? { |filter| filter.filter?(record) }
+        changed(true); notify_observers(:record_filtered, record)
+        return
+      end
+      @parser_block.call(@create_event_proc, record)
+      if @event_builder.count == 0
+        changed(true); notify_observers(:record_missed, record)
+        # Keep track of the fact that this line did not generate any events?
+      else 
+        @event_builder.each_built_event do |event|
+          events << event
+          @output_formatter.handle_event(event)
+        end
+        changed(true); notify_observers(:record_parsed, record, events)
+      end
+      return
+    ensure 
+      changed(true); notify_observers(:end_parse_record, record)
+    end
+
+    # Runs a feed
+    # @param [Threatinator::Feed] feed The feed to run
+    # @param [Threatinator::Output] output The output instance
+    # @param [Hash] run_opts Options passed to #run. See #run .
+    def self.run(feed, output, run_opts = {})
+      self.new(feed, output).run(run_opts)
+    end
+
+  end
+end

data/lib/threatinator/fetcher.rb

@@ -0,0 +1,22 @@
+module Threatinator
+  class Fetcher
+
+    # @param [Hash] opts An options hash. See subclasses for details.
+    def initialize(opts = {})
+    end
+
+    # @return [IO] an IO object
+    def fetch
+      raise NotImplementedError.new("#{self.class}#fetch not implemented!")
+    end
+
+    def ==(other)
+      true
+    end
+
+    def eql?(other)
+      self.class == other.class &&
+        self == other
+    end
+  end
+end

data/lib/threatinator/fetchers/http.rb

@@ -0,0 +1,46 @@
+require 'threatinator/fetcher'
+require 'threatinator/exceptions'
+require 'typhoeus'
+require 'tempfile'
+
+module Threatinator
+  module Fetchers
+    class Http < Threatinator::Fetcher
+      attr_reader :url
+      # @param [Hash] opts An options hash. 
+      # @option opts [Addressable::URI] :url The URL that is to be fetched 
+      #   (required)
+      #
+      def initialize(opts = {})
+        @url = opts.delete(:url) or raise ArgumentError.new("Missing :url")
+        super(opts)
+      end
+
+      def ==(other)
+        @url == other.url && super(other)
+      end
+
+      # @return [IO] an IO-style object.
+      # @raise [Threatinator::Exceptions::FetchFailed] if the fetch fails
+      def fetch
+        tempfile = Tempfile.new("threatinator_http")
+        request = Typhoeus::Request.new(@url, ssl_verifypeer: false)
+        request.on_headers do |response|
+          if response.response_code != 200
+
+            raise Threatinator::Exceptions::FetchFailed.new("Request failed!")
+          end
+        end
+        request.on_body do |chunk|
+          tempfile.write(chunk)
+        end
+        # Run it
+        request.run
+        # Reset the IO to the beginning of the file
+        tempfile.rewind
+        tempfile
+      end
+    end
+
+  end
+end

data/lib/threatinator/filter.rb

@@ -0,0 +1,12 @@
+module Threatinator
+  # Acts as a filter for parser data.
+  class Filter
+    # What is passed in as arguments depends upon the parser. 
+    #
+    # @param [Threatinator::Record] record The record to filter
+    # @return [Boolean] true if filtered, false otherwise.
+    def filter?(record)
+      raise NotImplementedError.new("#{self.class}.filter?(record) not implemented")
+    end
+  end
+end

data/lib/threatinator/filters/block.rb

@@ -0,0 +1,18 @@
+require 'threatinator/filter'
+
+module Threatinator
+  module Filters
+    # Basic filter that allows for arbitrary filtering.
+    class Block < Threatinator::Filter
+      def initialize(block)
+        @block = block
+      end
+
+      # @param [Threatinator::Record] record The record to filter
+      # @return [Boolean] true if filtered, false otherwise.
+      def filter?(record)
+        !! @block.call(record)
+      end
+    end
+  end
+end

data/lib/threatinator/filters/comments.rb

@@ -0,0 +1,16 @@
+require 'threatinator/filter'
+
+module Threatinator
+  module Filters
+    # Filters out any lines of text that begin with a comment '#'
+    class Comments < Threatinator::Filter
+      # @param [Threatinator::Record] record The record to filter
+      # @return [Boolean] true if filtered, false otherwise.
+      def filter?(record)
+        record.data[0] == '#'
+      end
+    end
+  end
+end
+
+

data/lib/threatinator/filters/whitespace.rb

@@ -0,0 +1,19 @@
+require 'threatinator/filter'
+
+module Threatinator
+  module Filters
+    # Filters on any lines of text that consist entirely of whitespace
+    class Whitespace < Threatinator::Filter
+      def initialize()
+        @re = /^\s*$/
+      end
+
+      # @param [Threatinator::Record] record The record to filter
+      # @return [Boolean] true if filtered, false otherwise.
+      def filter?(record)
+        !! @re.match(record.data)
+      end
+    end
+  end
+end
+

data/lib/threatinator/output.rb

@@ -0,0 +1,50 @@
+require 'threatinator/config/base'
+
+module Threatinator
+  class Output
+    def initialize(config)
+    end
+
+    def handle_event(event)
+      #:nocov:
+      raise NotImplementedError.new("#{self.class}#handle_event is not implemented")
+      #:nocov:
+    end
+
+    def finish
+      #:nocov:
+      raise NotImplementedError.new("#{self.class}#finish is not implemented")
+      #:nocov:
+    end
+
+    class Config < Threatinator::Config::Base
+    end
+  end
+
+  class FileBasedOutput < Output
+    attr_reader :output_io
+    protected :output_io
+
+    def initialize(config)
+      super(config)
+      if io = config.io
+        @output_io = io
+      elsif filename = config.filename
+        @output_io = File.open(filename, 'w:UTF-8')
+      else
+        @output_io = $stdout.dup
+      end
+    end
+
+    def finish
+      @output_io.close
+    end
+
+    class Config < superclass::Config
+      attribute :filename, String, 
+        description: "Path to the file where output will be written"
+
+      attribute :io
+    end
+  end
+end

data/lib/threatinator/parser.rb

@@ -0,0 +1,23 @@
+module Threatinator
+  class Parser
+    # @param [Hash] opts An options hash. See subclasses for details.
+    def initialize(opts = {})
+    end
+
+    # Runs the parser against the provided io, yielding records.
+    # @param [IO] io The IO to be parsed.
+    def run(io)
+      raise NotImplementedError.new("#{self.class}#run not implemented!")
+    end
+
+    def ==(other)
+      true
+    end
+
+    def eql?(other)
+      self.class == other.class &&
+        self == other
+    end
+  end
+end
+

data/lib/threatinator/parsers/csv.rb

@@ -0,0 +1,7 @@
+module Threatinator
+  module Parsers
+    module CSV
+      require 'threatinator/parsers/csv/parser'
+    end
+  end
+end

data/lib/threatinator/parsers/csv/parser.rb

@@ -0,0 +1,77 @@
+require 'threatinator/record'
+require 'threatinator/parser'
+require 'csv'
+
+module Threatinator
+  module Parsers
+    module CSV
+      # Parses an IO, yielding a record with a CSV::Row.
+      class Parser < Threatinator::Parser
+        attr_reader :csv_opts, :row_separator, :col_separator, :headers
+
+        # @param [Hash] opts 
+        # @option opts [String, :auto] :row_separator A string that represent the row
+        #  separator. Identical to ::CSV.new's :row_sep.
+        # @option opts [String] :col_separator A string that represents the column
+        #  separator. Identical to ::CSV.new's :col_sep.
+        # @option opts [Array<String>, :first_row, true, false] :headers The header
+        #  configuration. Identical to ::CSV.new's :headers. 
+        # @option opts [Hash] :csv_opts A hash of options that will be passed to
+        #  Ruby's CSV.new. 
+        # @see ::CSV
+        def initialize(opts = {})
+          @csv_opts = {}.merge(opts.delete(:csv_opts) || {})
+          @row_separator = opts.delete(:row_separator)
+          @col_separator = opts.delete(:col_separator)
+          @headers = opts.delete(:headers)
+
+          super(opts)
+        end
+
+        def ==(other)
+          @csv_opts == other.csv_opts &&
+            @row_separator == other.row_separator &&
+            @col_separator == other.col_separator &&
+            @headers == other.headers &&
+            super(other)
+        end
+
+        def _build_csv_opts
+          opts = {}.merge(@csv_opts)
+          opts[:return_headers] = true
+          opts[:row_sep] = @row_separator unless @row_separator.nil?
+          opts[:col_sep] = @col_separator unless @col_separator.nil?
+          opts[:headers] = @headers unless @headers.nil?
+          opts
+        end
+
+        # @param [IO] io
+        # @yield [record] Gives one line to the block
+        # @yieldparam record [Record] a record
+        def run(io)
+          lineno = 1
+          previous_pos = io.pos
+          csv = ::CSV.new(io, _build_csv_opts())
+          csv.each do |row|
+            begin
+              if row.kind_of?(::CSV::Row)
+                next if row.header_row?
+                row = row.to_hash
+              end
+
+              yield Record.new(row, 
+                               line_number: lineno, 
+                               pos_start: previous_pos, 
+                               pos_end: io.pos)
+
+            ensure
+              previous_pos = io.pos
+              lineno += 1
+            end
+          end
+          nil
+        end
+      end
+    end
+  end
+end