threatinator 0.1.1

data/lib/threatinator/cli/run_action_builder.rb

@@ -0,0 +1,41 @@
+require 'threatinator/cli/action_builder'
+require 'threatinator/actions/run'
+require 'threatinator/actions/run/coverage_observer'
+require 'csv'
+
+module Threatinator
+  module CLI
+    class RunActionBuilder < ActionBuilder
+      def initialize(opts, args, config_class)
+        super(opts, args)
+        @config_class = config_class
+      end
+
+      def build
+        Threatinator::Actions::Run::Action.new(feed_registry, config)
+      end
+
+      def config
+        run_hash = config_hash["run"] || {}
+        run_hash['observers'] ||= []
+
+        if filename = run_hash['coverage_output']
+          observer = Threatinator::Actions::Run::CoverageObserver.new(filename)
+          run_hash['observers'] << observer
+        end
+
+        config = @config_class.new(run_hash)
+
+        if config.feed_provider.nil? && provider = extra_args.shift
+          config.feed_provider = provider
+        end
+
+        if config.feed_name.nil? && name = extra_args.shift
+          config.feed_name = name
+        end
+        config
+      end
+
+    end
+  end
+end

data/lib/threatinator/config.rb

@@ -0,0 +1,6 @@
+module Threatinator
+  module Config
+    require 'threatinator/config/feed_search'
+  end
+end
+

data/lib/threatinator/config/base.rb

@@ -0,0 +1,35 @@
+require 'virtus'
+module Threatinator
+  module Config
+    class Base
+      include Virtus.model
+
+      def self.properties(namespace = nil)
+        ret = {}
+        self.attribute_set.each do |attribute|
+          name = attribute.name.to_s
+          unless namespace.nil?
+            name = [namespace, name].join('.')
+          end
+
+          if attribute.primitive.ancestors.include?(Threatinator::Config::Base)
+            ret.merge!(attribute.primitive.properties(name))
+            next
+          end
+
+          desc = attribute.options[:description]
+          case desc
+          when nil
+            next
+          when ::Proc
+            desc = desc.call(self, attribute)
+          end 
+          ret[name] = [desc, attribute.type]
+        end
+        ret
+      end
+    end
+
+  end
+end
+

data/lib/threatinator/config/feed_search.rb

@@ -0,0 +1,25 @@
+require 'threatinator/config/base'
+
+module Threatinator
+  module Config
+    class FeedSearch < Threatinator::Config::Base
+      DEFAULT_FEED_PATH = File.expand_path("../../../../feeds", __FILE__)
+
+      attribute :exclude_default, Boolean, default: false, 
+        description: 'Exclude default path from feed search path'
+
+      attribute :path, Array[String],
+        description: 'The paths to search for feeds'
+
+      # @return [Array<String>] An array of paths to search
+      def search_path
+        ret = self.path
+        if self.exclude_default == false
+          ret = ret + [DEFAULT_FEED_PATH]
+        end
+        ret
+      end
+    end
+  end
+end
+

data/lib/threatinator/decoder.rb

@@ -0,0 +1,24 @@
+module Threatinator
+  # Decodes/Extracts data from an input IO, producing a new IO. The decoder is
+  # initialized with a configuration, and then #decode is called upon an IO
+  # object.
+  class Decoder
+    attr_reader :encoding
+
+    # @param [Hash] opts An options hash
+    # @option opts [String] :encoding The encoding for the output IO. Defaults
+    #  to "utf-8"
+    def initialize(opts = {})
+      @encoding = opts[:encoding] || "utf-8"
+    end
+
+    # Decodes an input IO, returning a brand new IO.
+    # @param [IO] io The IO to decode
+    # @return [IO] A new IO.
+    def decode(io)
+      #:nocov:
+      raise NotImplementedError.new("#{self.class}#decode not implemented!")
+      #:nocov:
+    end
+  end
+end

data/lib/threatinator/decoders/gzip.rb

@@ -0,0 +1,30 @@
+require 'threatinator/decoder'
+require 'threatinator/exceptions'
+require 'zlib'
+require 'tempfile'
+require 'pp'
+
+module Threatinator
+  module Decoders
+    class Gzip < Threatinator::Decoder
+      # Decompresses the io using Gzip.
+      # @param (see Threatinator::Decoder#decode)
+      def decode(io)
+        zio = Zlib::GzipReader.new(io, encoding: "binary")
+        tempfile = Tempfile.new("threatinator", encoding: "binary")
+        while chunk = zio.read(10240)
+          tempfile.write(chunk)
+        end
+        
+        zio.close
+        io.close unless io.closed?
+        tempfile.rewind
+        tempfile.set_encoding(self.encoding)
+        tempfile
+      rescue Zlib::GzipFile::Error => e
+        raise Threatinator::Exceptions::DecoderError.new
+      end
+
+    end
+  end
+end

data/lib/threatinator/event.rb

@@ -0,0 +1,27 @@
+require 'threatinator/property_definer'
+
+module Threatinator
+  class Event
+    include Threatinator::PropertyDefiner
+
+    VALID_TYPES = Set.new([:c2, :attacker, :malware_host, :spamming, :scanning, :phishing])
+
+    def initialize(opts = {})
+      _parse_properties(opts)
+    end
+
+    property :feed_provider, type: String
+    property :feed_name, type: String
+    property :type, type: Symbol, validate: lambda { |obj, val| VALID_TYPES.include?(val) }
+    property :ipv4s, type: Array, default: lambda { Array.new }
+    property :fqdns, type: Array, default: lambda { Array.new }
+
+    def add_ipv4(ipv4)
+      self.ipv4s << ipv4
+    end
+
+    def add_fqdn(fqdn)
+      self.fqdns << fqdn
+    end
+  end
+end

data/lib/threatinator/event_builder.rb

@@ -0,0 +1,41 @@
+require 'threatinator/event'
+
+module Threatinator
+  class EventBuilder
+    attr_reader :total
+    def initialize(feed)
+      @feed = feed
+      @built_events = []
+      @total = 0
+    end
+
+    def each_built_event
+      @built_events.each do |event|
+        yield event
+      end
+      @built_events.clear
+    end
+
+    def count
+      @built_events.count
+    end
+
+    def clear
+      @built_events.clear
+    end
+
+    def create_event_proc
+      self.method(:create_event).to_proc
+    end
+
+    def create_event
+      event = Threatinator::Event.new
+      event.feed_provider = @feed.provider
+      event.feed_name = @feed.name
+      yield(event)
+      @total += 1
+      @built_events << event
+    end
+  end
+end
+

data/lib/threatinator/exceptions.rb

@@ -0,0 +1,61 @@
+module Threatinator
+  module Exceptions
+    # Indicates that a fetch failed.
+    class FetchFailed < StandardError
+    end
+    
+    # Indicates that the decode operation failed
+    class DecoderError < StandardError
+    end
+
+    class ParseError < StandardError
+    end
+
+    class PluginLoadError < StandardError
+      attr_reader :cause
+      def initialize(message, cause = nil)
+        @cause = cause
+        unless cause.nil?
+          message = "#{message} : #{cause.class} : #{cause}"
+          self.set_backtrace cause.backtrace
+        end
+        super(message)
+      end
+    end
+
+    class UnknownPlugin < StandardError
+    end
+
+    class CouldNotFindOutputConfigError < StandardError
+    end
+    
+    class InvalidAttributeError < StandardError
+      attr_reader :attribute, :got
+      def initialize(attribute, got)
+        @attribute = attribute
+        @got = got
+        super("Invalid value for attribute '#{attribute}'. Got " + got.inspect)
+      end
+    end
+
+    class AlreadyRegisteredError < StandardError
+    end
+
+    class UnknownFeed < StandardError
+      attr_reader :provider, :name
+      def initialize(provider, name)
+        @provider = provider
+        @name = name
+        super("Failed to find feed with provider '#{provider}' and name '#{name}'")
+      end
+    end
+
+    class FeedFileNotFoundError < StandardError
+      def initialize(filename)
+        @filename = filename
+        super("Failed to open/read feed file '#{filename}'")
+      end
+    end
+  end
+end
+

data/lib/threatinator/feed.rb

@@ -0,0 +1,82 @@
+require 'threatinator/exceptions'
+
+module Threatinator
+  class Feed
+    # @param [Hash] opts Options hash
+    # @option opts [String] :provider The name of the provider
+    # @option opts [String] :name The name of the feed
+    # @option opts [Proc] :parser_block A block that will be called by the 
+    #   parser each time it processes a record.
+    # @option opts [Proc] :parser_builder A proc that, when called, will 
+    #   return a brand new instance of a Threatinator::Parser.
+    # @option opts [Proc] :fetcher_builder A proc that, when called, will 
+    #   return a brand new instance of a Threatinator::Fetcher.
+    # @option opts [Array<Proc>] :filter_builders An array of procs that, 
+    #   when called, will each return an instance of a filter (something that 
+    #   responds to :filter?)
+    # @option opts [Array<Proc>] :decoder_builders An array of procs that, 
+    #   when called, will each return an instance of a Threatinator::Decoder
+    def initialize(opts = {})
+      @provider = opts.delete(:provider)
+      @name = opts.delete(:name)
+      @parser_block = opts.delete(:parser_block)
+
+      @parser_builder = opts.delete(:parser_builder)
+      @fetcher_builder = opts.delete(:fetcher_builder)
+      @filter_builders = opts.delete(:filter_builders) || []
+      @decoder_builders = opts.delete(:decoder_builders) || []
+      validate!
+    end
+
+    def provider
+      @provider.dup
+    end
+
+    def name
+      @name.dup
+    end
+
+    def parser_block
+      @parser_block
+    end
+
+    def fetcher_builder
+      @fetcher_builder
+    end
+
+    def parser_builder
+      @parser_builder
+    end
+
+    def filter_builders
+      @filter_builders.dup
+    end
+
+    def decoder_builders
+      @decoder_builders.dup
+    end
+
+    def validate!
+      validate_attribute!(:provider, @provider) { |x| x.kind_of?(::String) }
+      validate_attribute!(:name, @name) { |x| x.kind_of?(::String) }
+      validate_attribute!(:parser_block, @parser_block) { |x| x.kind_of?(::Proc) }
+      validate_attribute!(:fetcher_builder, @fetcher_builder) { |x| x.kind_of?(::Proc) }
+      validate_attribute!(:parser_builder, @parser_builder) { |x| x.kind_of?(::Proc) }
+      validate_attribute!(:filter_builders, @filter_builders) do |x|
+        x.kind_of?(::Array) &&
+          x.all? { |e| e.kind_of?(::Proc) }
+      end
+
+      validate_attribute!(:decoder_builders, @decoder_builders) do |x|
+        x.kind_of?(::Array) &&
+          x.all? { |e| e.kind_of?(::Proc) }
+      end
+    end
+
+    def validate_attribute!(name, val, &block)
+      unless block.call(val) == true
+        raise Threatinator::Exceptions::InvalidAttributeError.new(name, val)
+      end
+    end
+  end
+end

data/lib/threatinator/feed_builder.rb

@@ -0,0 +1,156 @@
+require 'docile'
+require 'threatinator/feed'
+require 'threatinator/exceptions'
+require 'threatinator/decoders/gzip'
+require 'threatinator/fetchers/http'
+require 'threatinator/parsers/getline'
+require 'threatinator/parsers/csv'
+require 'threatinator/parsers/json'
+require 'threatinator/parsers/xml'
+require 'threatinator/filters/block'
+require 'threatinator/filters/whitespace'
+require 'threatinator/filters/comments'
+
+module Threatinator
+  class FeedBuilder
+    def provider(provider_name)
+      @provider = provider_name
+      self
+    end
+
+    def name(name)
+      @name = name
+      self
+    end
+
+    def fetch_http(url, opts = {})
+      opts[:url] = url
+      @fetcher_builder = lambda do
+        opts_dup = Marshal.load(Marshal.dump(opts))
+        Threatinator::Fetchers::Http.new(opts_dup)
+      end
+      self
+    end
+
+  def parse_xml(pattern_string, opts = {}, &block)
+    @parser_builder = lambda do
+      pattern = Threatinator::Parsers::XML::Pattern.new(pattern_string)
+      opts_dup = Marshal.load(Marshal.dump(opts))
+      opts_dup[:pattern] = pattern
+      Threatinator::Parsers::XML::Parser.new(opts_dup, &block)
+    end
+    @parser_block = block
+    self
+  end
+
+    def parse_json(opts = {}, &block)
+      @parser_builder = lambda do
+        opts_dup = Marshal.load(Marshal.dump(opts))
+        Threatinator::Parsers::JSON::Parser.new(opts_dup, &block)
+      end
+      @parser_block = block
+      self
+    end
+
+    def parse_eachline(opts = {}, &block)
+      @parser_builder = lambda do
+        opts_dup = Marshal.load(Marshal.dump(opts))
+        Threatinator::Parsers::Getline::Parser.new(opts_dup, &block)
+      end
+      @parser_block = block
+      self
+    end
+
+    def parse_csv(opts = {}, &block)
+      @parser_builder = lambda do
+        opts_dup = Marshal.load(Marshal.dump(opts))
+        Threatinator::Parsers::CSV::Parser.new(opts_dup, &block)
+      end
+      @parser_block = block
+      self
+    end
+
+    # Specify a block filter for the parser
+    def filter(&block)
+      @filter_builders ||= []
+      @filter_builders << lambda { Threatinator::Filters::Block.new(block) }
+      self
+    end
+
+    # Filter out whitespace lines. Only works on line-based text.
+    def filter_whitespace
+      @filter_builders ||= []
+      @filter_builders << lambda { Threatinator::Filters::Whitespace.new }
+      self
+    end
+    
+    # Filter out whitespace lines. Only works on line-based text.
+    def filter_comments
+      @filter_builders ||= []
+      @filter_builders << lambda { Threatinator::Filters::Comments.new }
+      self
+    end
+
+    # Add the Gzip decoder
+    def decode_gzip
+      decoder_builders << lambda { Threatinator::Decoders::Gzip.new }
+      self
+    end
+    alias_method :extract_gzip, :decode_gzip
+    alias_method :gunzip, :decode_gzip
+
+    def decoder_builders
+      @decoder_builders ||= []
+    end
+    private :decoder_builders
+
+    def build
+      Feed.new(
+        :provider => @provider, 
+        :name => @name,
+        :parser_block => @parser_block,
+        :fetcher_builder => @fetcher_builder,
+        :parser_builder => @parser_builder,
+        :filter_builders => @filter_builders,
+        :decoder_builders => decoder_builders
+      )
+    end
+
+    # Loads the provided file, and generates a builder from it.
+    # @param [String] filename The name of the file to read the feed from
+    # @raise [FeedFileNotFoundError] if the file is not found
+    def self.from_file(filename)
+      begin 
+        filedata = File.read(filename)
+      rescue Errno::ENOENT
+        raise Threatinator::Exceptions::FeedFileNotFoundError.new(filename)
+      end
+      from_string(filedata, filename, 0)
+    end
+
+    # Generates a builder from a string via eval.
+    # @param [String] str The DSL code that specifies the feed.
+    # @param [String] filename (nil) Passed to eval. 
+    # @param [String] lineno (nil) Passed to eval. 
+    # @raise [FeedFileNotFoundError] if the file is not found
+    # @see Kernel#eval for details on filename and lineno
+    def self.from_string(str, filename = nil, lineno = nil)
+      from_dsl do
+        args = [str, binding]
+        unless filename.nil?
+          args << filename
+          unless lineno.nil?
+            args << lineno
+          end
+        end
+        eval(*args)
+      end
+    end
+
+    # Executes the block parameter within DSL scope
+    def self.from_dsl(&block)
+      Docile.dsl_eval(self.new, &block)
+    end
+  end
+end
+