shadowbq-threatinator 0.5.0

data/lib/threatinator/actions/run/output_config.rb

@@ -0,0 +1,59 @@
+require 'threatinator/config/base'
+require 'threatinator/exceptions'
+
+module Threatinator
+  module Actions
+    module Run
+      module OutputConfigClassMethods
+        def set_plugin_loader(pl)
+          @plugin_loader = pl
+          pl.each(:output) do |type, name, plugin|
+            self.attribute name, plugin::Config, default: lambda { |c,a| plugin::Config.new }
+          end
+        end
+
+        def get_plugin(name)
+          @plugin_loader.get(:output, name)
+        end
+
+        def formats
+          @plugin_loader.each(:output).map { |t, k, p| k.to_s }
+        end
+
+        def formats_str
+          formats.sort.join(', ')
+        end
+
+      end
+
+      module OutputConfigMethods
+        def build_output
+          oc = self.class.get_plugin(self.format)
+          if oc.nil?
+            raise Threatinator::Exceptions::UnknownPlugin.new("Unknown output plugin: '#{format}'")
+          end
+          output_config = self[format]
+
+          if output_config.nil?
+            raise Threatinator::Exceptions::CouldNotFindOutputConfigError.new("Could not find output config for '#{format}'. Perhaps there's some load-order issues?")
+          end
+
+          oc.new(output_config)
+        end
+      end
+
+      module OutputConfig
+        def self.generate(plugin_loader)
+          anonymous_class = Class.new(Threatinator::Config::Base) do
+            extend OutputConfigClassMethods
+            include OutputConfigMethods
+            set_plugin_loader(plugin_loader)
+            attribute :format, Symbol, default: lambda { |c,a| :csv },
+              description: lambda { |c, a| "Output format (#{c.formats_str})" }
+          end
+          anonymous_class
+        end
+      end
+    end
+  end
+end

data/lib/threatinator/actions/run/status_observer.rb

@@ -0,0 +1,37 @@
+module Threatinator
+  module Actions
+    module Run
+      class StatusObserver
+        attr_reader :missed, :filtered, :parsed, :errors
+        def initialize
+          @missed = @filtered = @parsed = @errors = 0
+        end
+
+        def total
+          @missed + @filtered + @parsed + @errors
+        end
+
+        # Handles FeedRunner observations
+        def update(message, *args)
+          case message
+          when :record_missed
+            @missed += 1
+          when :record_filtered
+            @filtered += 1
+          when :record_parsed
+            @parsed += 1
+          when :record_error
+            @errors += 1
+          end
+        end
+
+        def missed?; @missed > 0; end
+        def parsed?; @parsed > 0; end
+        def filtered?; @filtered > 0; end
+        def errors?; @errors > 0; end
+
+      end
+    end
+  end
+end
+

data/lib/threatinator/actions/run.rb

@@ -0,0 +1,2 @@
+require 'threatinator/actions/run/action'
+require 'threatinator/actions/run/config'

data/lib/threatinator/cli/action_builder.rb

@@ -0,0 +1,33 @@
+require 'threatinator/config'
+require 'threatinator/feed_registry'
+
+module Threatinator
+  module CLI
+    class ActionBuilder
+      attr_reader :extra_args, :config_hash
+
+      def initialize(config_hash, extra_args)
+        @extra_args = extra_args
+        @config_hash = config_hash
+        @feed_registry = nil
+      end
+
+      def build
+        #:nocov:
+        raise NotImplementedError.new("#{self.class}#build not implemented")
+        #:nocov:
+      end
+
+      def feed_registry
+        return @feed_registry unless @feed_registry.nil?
+
+        feed_search_hash = config_hash["feed_search"] || {}
+        feed_search_config = Threatinator::Config::FeedSearch.new(feed_search_hash)
+
+        @feed_registry = Threatinator::FeedRegistry.build(feed_search_config)
+      end
+    end
+
+  end
+end
+

data/lib/threatinator/cli/list_action_builder.rb

@@ -0,0 +1,19 @@
+require 'threatinator/cli/action_builder'
+require 'threatinator/actions/list'
+
+module Threatinator
+  module CLI
+    class ListActionBuilder < ActionBuilder
+      def build
+        Threatinator::Actions::List::Action.new(feed_registry, config)
+      end
+
+      def config
+        list_hash = config_hash["list"] || {}
+        Threatinator::Actions::List::Config.new(list_hash)
+      end
+
+    end
+  end
+end
+

data/lib/threatinator/cli/parser.rb

@@ -0,0 +1,123 @@
+require 'gli'
+require 'threatinator/actions/run'
+require 'threatinator/actions/list'
+require 'threatinator/config/feed_search'
+require 'threatinator/config/logger'
+require 'threatinator/cli/list_action_builder'
+require 'threatinator/cli/run_action_builder'
+require 'threatinator/plugin_loader'
+
+module Threatinator
+  module CLI
+    module Helpers
+      def add_cli_args(gli, config_properties)
+        config_properties.each do |key, args|
+          desc, type, default_value = args
+          if type.base == Axiom::Types::Boolean
+            gli.switch key, desc: desc
+          elsif type.base ==Axiom::Types::Array
+            gli.flag key, desc: desc, type: Array
+          else 
+            gli.flag key, desc: desc
+          end
+        end
+      end
+
+      def clean_opts(opts)
+        opts.delete_if {|k, v| k.kind_of?(::Symbol) or (k == 'help') or v.nil? }
+        opts
+      end
+
+      def nest_opts(opts)
+        config_hash = {}
+        opts.each_pair do |key, val|
+          key_path = key.to_s.split('.')
+          final_key = key_path.pop
+          nested_hash = config_hash
+          while key_path.length > 0
+            part = key_path.shift
+            nested_hash[part] ||= {}
+            nested_hash = nested_hash[part]
+          end
+          nested_hash[final_key] = val
+        end
+        config_hash
+      end
+
+      def fix_opts(opts)
+        nest_opts(clean_opts(opts))
+      end
+    end
+
+    class Parser
+      include Helpers
+
+      attr_accessor :builder
+      attr_reader :config_hash, :extra_args
+      attr_reader :run_action_config_class
+
+      def initialize
+        @builder = nil
+        @plugin_loader = Threatinator::PluginLoader.new
+        @plugin_loader.load_all_plugins
+        @run_action_config_class = Threatinator::Actions::Run::Config.generate(@plugin_loader)
+        @mod = _init_mod
+      end
+
+      def parse(args)
+        @mod.run(args)
+      end
+
+      def set_opts(global_options, options, args)
+        @config_hash = fix_opts(global_options.merge(options))
+        @extra_args = args
+      end
+
+      def _init_mod
+        parser = self
+        Module.new do
+          extend Helpers
+          extend GLI::App
+
+          program_desc 'Threatinator!'
+
+          add_cli_args(self, Threatinator::Config::Logger.properties('logger'))
+          add_cli_args(self, Threatinator::Config::FeedSearch.properties('feed_search'))
+
+          desc "fetch and parse a feed"
+          command :run do |c|
+            c.flag 'run.coverage_output', type: String, desc: "Write coverage analysis to the specified file (CSV format)"
+
+            add_cli_args(c, parser.run_action_config_class.properties('run'))
+            c.action do |global_options, options, args|
+              if options["run.feed_provider"].nil?
+                options["run.feed_provider"] = args.shift
+              end
+
+              if options["run.feed_name"].nil?
+                options["run.feed_name"] = args.shift
+              end
+
+              parser.set_opts(global_options, options, args)
+
+              builder = Threatinator::CLI::RunActionBuilder.new(parser.config_hash, parser.extra_args, parser.run_action_config_class)
+              parser.builder = builder
+            end
+          end
+
+          desc 'list out all the feeds'
+          command :list do |c|
+            add_cli_args(c, Threatinator::Actions::List::Config.properties('list'))
+            c.action do |global_options, options, args|
+              parser.set_opts(global_options, options, args)
+              parser.builder = Threatinator::CLI::ListActionBuilder.new(parser.config_hash, parser.extra_args)
+            end
+          end
+        end
+      end
+
+    end # _init_mod
+  end
+
+end
+

data/lib/threatinator/cli/run_action_builder.rb

@@ -0,0 +1,41 @@
+require 'threatinator/cli/action_builder'
+require 'threatinator/actions/run'
+require 'threatinator/actions/run/coverage_observer'
+require 'csv'
+
+module Threatinator
+  module CLI
+    class RunActionBuilder < ActionBuilder
+      def initialize(opts, args, config_class)
+        super(opts, args)
+        @config_class = config_class
+      end
+
+      def build
+        Threatinator::Actions::Run::Action.new(feed_registry, config)
+      end
+
+      def config
+        run_hash = config_hash["run"] || {}
+        run_hash['observers'] ||= []
+
+        if filename = run_hash['coverage_output']
+          observer = Threatinator::Actions::Run::CoverageObserver.new(filename)
+          run_hash['observers'] << observer
+        end
+
+        config = @config_class.new(run_hash)
+
+        if config.feed_provider.nil? && provider = extra_args.shift
+          config.feed_provider = provider
+        end
+
+        if config.feed_name.nil? && name = extra_args.shift
+          config.feed_name = name
+        end
+        config
+      end
+
+    end
+  end
+end

data/lib/threatinator/cli.rb

@@ -0,0 +1,19 @@
+require 'threatinator/cli/parser'
+require 'threatinator/config/logger'
+require 'threatinator/logger'
+module Threatinator
+  module CLI
+    def self.process!(args)
+      parser = Parser.new
+      ret = parser.parse(args)
+      builder = parser.builder
+      return ret if builder.nil?
+
+      conf = Threatinator::Config::Logger.new(parser.config_hash['logger'])
+      Threatinator::Logger.configure_logger(conf)
+
+      builder.build.exec
+      ret
+    end
+  end
+end

data/lib/threatinator/config/base.rb

@@ -0,0 +1,35 @@
+require 'virtus'
+module Threatinator
+  module Config
+    class Base
+      include Virtus.model
+
+      def self.properties(namespace = nil)
+        ret = {}
+        self.attribute_set.each do |attribute|
+          name = attribute.name.to_s
+          unless namespace.nil?
+            name = [namespace, name].join('.')
+          end
+
+          if attribute.primitive.ancestors.include?(Threatinator::Config::Base)
+            ret.merge!(attribute.primitive.properties(name))
+            next
+          end
+
+          desc = attribute.options[:description]
+          case desc
+          when nil
+            next
+          when ::Proc
+            desc = desc.call(self, attribute)
+          end 
+          ret[name] = [desc, attribute.type]
+        end
+        ret
+      end
+    end
+
+  end
+end
+

data/lib/threatinator/config/feed_search.rb

@@ -0,0 +1,25 @@
+require 'threatinator/config/base'
+
+module Threatinator
+  module Config
+    class FeedSearch < Threatinator::Config::Base
+      DEFAULT_FEED_PATH = File.expand_path("../../../../feeds", __FILE__)
+
+      attribute :exclude_default, Boolean, default: false, 
+        description: 'Exclude default path from feed search path'
+
+      attribute :path, Array[String],
+        description: 'The paths to search for feeds'
+
+      # @return [Array<String>] An array of paths to search
+      def search_path
+        ret = self.path
+        if self.exclude_default == false
+          ret = ret + [DEFAULT_FEED_PATH]
+        end
+        ret
+      end
+    end
+  end
+end
+

data/lib/threatinator/config/logger.rb

@@ -0,0 +1,14 @@
+require 'threatinator/config/base'
+require 'threatinator/logger'
+
+module Threatinator
+  module Config
+    class Logger < Threatinator::Config::Base
+      attribute :level, String, default: Threatinator::Logger.level_string, 
+        coercer: lambda { |v| v.to_s.upcase },
+        description: "Set the logging level: #{::Threatinator::Logger.levels.join(',')}"
+    end
+  end
+end
+
+

data/lib/threatinator/config.rb

@@ -0,0 +1,7 @@
+module Threatinator
+  module Config
+    require 'threatinator/config/feed_search'
+    require 'threatinator/config/logger'
+  end
+end
+

data/lib/threatinator/decoder.rb

@@ -0,0 +1,24 @@
+module Threatinator
+  # Decodes/Extracts data from an input IO, producing a new IO. The decoder is
+  # initialized with a configuration, and then #decode is called upon an IO
+  # object.
+  class Decoder
+    attr_reader :encoding
+
+    # @param [Hash] opts An options hash
+    # @option opts [String] :encoding The encoding for the output IO. Defaults
+    #  to "utf-8"
+    def initialize(opts = {})
+      @encoding = opts[:encoding] || "utf-8"
+    end
+
+    # Decodes an input IO, returning a brand new IO.
+    # @param [IO] io The IO to decode
+    # @return [IO] A new IO.
+    def decode(io)
+      #:nocov:
+      raise NotImplementedError.new("#{self.class}#decode not implemented!")
+      #:nocov:
+    end
+  end
+end

data/lib/threatinator/decoders/gzip.rb

@@ -0,0 +1,30 @@
+require 'threatinator/decoder'
+require 'threatinator/exceptions'
+require 'zlib'
+require 'tempfile'
+require 'pp'
+
+module Threatinator
+  module Decoders
+    class Gzip < Threatinator::Decoder
+      # Decompresses the io using Gzip.
+      # @param (see Threatinator::Decoder#decode)
+      def decode(io)
+        zio = Zlib::GzipReader.new(io, encoding: "binary")
+        tempfile = Tempfile.new("threatinator", encoding: "binary")
+        while chunk = zio.read(10240)
+          tempfile.write(chunk)
+        end
+        
+        zio.close
+        io.close unless io.closed?
+        tempfile.rewind
+        tempfile.set_encoding(self.encoding)
+        tempfile
+      rescue Zlib::GzipFile::Error => e
+        raise Threatinator::Exceptions::DecoderError.new
+      end
+
+    end
+  end
+end

data/lib/threatinator/event.rb

@@ -0,0 +1,63 @@
+require 'threatinator/model/base'
+require 'threatinator/model/observables/ipv4_collection'
+require 'threatinator/model/observables/fqdn_collection'
+require 'threatinator/model/observables/url_collection'
+require 'equalizer'
+require 'set'
+
+module Threatinator
+  class Event < Threatinator::Model::Base
+    include Equalizer.new(:feed_provider, :feed_name, :type, :ipv4s, :fqdns, :urls)
+    attr_reader :feed_provider, :feed_name, :type, :ipv4s, :fqdns, :urls
+
+    VALID_TYPES = Set.new([:c2, :attacker, :malware_host, :spamming, :scanning, :phishing])
+
+    validates :feed_provider, :feed_name, type: ::String
+    validates :type, type: ::Symbol, allow_nil: false, inclusion: {in: VALID_TYPES}
+    validates :ipv4s, type: Threatinator::Model::Observables::Ipv4Collection
+    validates :fqdns, type: Threatinator::Model::Observables::FqdnCollection
+    validates :urls, type: Threatinator::Model::Observables::UrlCollection
+
+    EventHeader = Struct.new(:feed_provider, :feed_name, :type)
+
+    # @param [Hash] opts
+    # @option opts [String] :feed_provider The name of the feed provider
+    # @option opts [String] :feed_name The name of the feed
+    # @option opts [Symbol] :type The 'type' of feed.
+    # @option opts [#each] :ipv4s A collection of ipv4s
+    # @option opts [#each] :fqdns A collection of FQDNs
+    # @option opts [#each] :urls A collection of Urls
+    def initialize(opts = {})
+      @feed_provider = opts[:feed_provider]
+      @feed_name = opts[:feed_name]
+      @type = opts[:type]
+      @ipv4s = Threatinator::Model::Observables::Ipv4Collection.new(opts[:ipv4s] || [])
+      @fqdns = Threatinator::Model::Observables::FqdnCollection.new(opts[:fqdns] || [])
+      @urls = Threatinator::Model::Observables::UrlCollection.new(opts[:urls] || [])
+      super()
+    end
+
+    def header
+      event_header = EventHeader.new(@feed_provider, @feed_name, @type)
+    end
+
+    def to_serializable_hash
+
+      ret = {
+        import_time: Time.now.utc.to_i,
+        feed_provider: @feed_provider,
+        feed_name: @feed_name,
+        source: 'threatinator'
+      }
+      if @type
+        ret[:tags] = @type.to_s
+      end
+
+      ret[:ipv4s] = @ipv4s.list
+      ret[:fqdns] = @fqdns.list
+      ret[:urls] = @urls.list
+      ret
+    end
+
+  end
+end

data/lib/threatinator/event_builder.rb

@@ -0,0 +1,70 @@
+require 'threatinator/event'
+require 'threatinator/exceptions'
+require 'threatinator/model/observables/ipv4'
+require 'ip'
+
+module Threatinator
+  class EventBuilder
+    attr_writer :type
+
+    def initialize(feed_provider, feed_name)
+      @feed_provider = feed_provider
+      @feed_name = feed_name
+      self.reset
+    end
+
+    def reset
+      @type = nil
+      @ipv4s = []
+      @fqdns = []
+      @urls = []
+    end
+
+    def build
+      opts = {
+        feed_provider: @feed_provider,
+        feed_name: @feed_name,
+      }
+      opts[:type] = @type unless @type.nil?
+
+      ret = Threatinator::Event.new(opts)
+
+      @ipv4s.each do |ipv4, opts|
+        opts = opts.dup
+        if ipv4.is_a?(::String)
+          ipv4 = ::IP::V4.parse(ipv4)
+        end
+        opts[:ipv4] = ipv4
+        ret.ipv4s << Threatinator::Model::Observables::Ipv4.new(opts)
+      end
+      @fqdns.each do |fqdn|
+        ret.fqdns << fqdn
+      end
+      @urls.each do |url|
+        url = begin
+          ::Addressable::URI.parse(url)
+        rescue TypeError => e
+          raise Threatinator::Exceptions::EventBuildError, "Failed to parse URL"
+        end
+        ret.urls << url
+      end
+      ret
+    rescue Threatinator::Exceptions::InvalidAttributeError => e
+      raise Threatinator::Exceptions::EventBuildError, e.message
+    end
+
+    def add_fqdn(fqdn)
+      @fqdns << fqdn
+    end
+
+    def add_ipv4(ipv4, opts = {})
+      @ipv4s << [ipv4, opts]
+    end
+
+    def add_url(url)
+      @urls << url
+    end
+  end
+end
+
+

data/lib/threatinator/exceptions.rb

@@ -0,0 +1,58 @@
+module Threatinator
+  module Exceptions
+    # Indicates that a fetch failed.
+    class FetchFailed < StandardError
+    end
+    
+    # Indicates that the decode operation failed
+    class DecoderError < StandardError
+    end
+
+    class ParseError < StandardError
+    end
+
+    class PluginLoadError < StandardError
+      attr_reader :cause
+      def initialize(message, cause = nil)
+        @cause = cause
+        unless cause.nil?
+          message = "#{message} : #{cause.class} : #{cause}"
+          self.set_backtrace cause.backtrace
+        end
+        super(message)
+      end
+    end
+
+    class UnknownPlugin < StandardError
+    end
+
+    class CouldNotFindOutputConfigError < StandardError
+    end
+    
+    class InvalidAttributeError < StandardError
+    end
+
+    class EventBuildError < StandardError
+    end
+
+    class AlreadyRegisteredError < StandardError
+    end
+
+    class UnknownFeed < StandardError
+      attr_reader :provider, :name
+      def initialize(provider, name)
+        @provider = provider
+        @name = name
+        super("Failed to find feed with provider '#{provider}' and name '#{name}'")
+      end
+    end
+
+    class FeedFileNotFoundError < StandardError
+      def initialize(filename)
+        @filename = filename
+        super("Failed to open/read feed file '#{filename}'")
+      end
+    end
+  end
+end
+