threatinator 0.1.1

data/lib/threatinator.rb

@@ -0,0 +1,3 @@
+module Threatinator
+  require 'threatinator/runner'
+end

data/lib/threatinator/action.rb

@@ -0,0 +1,14 @@
+module Threatinator
+  class Action
+    attr_reader :registry
+    def initialize(registry)
+      @registry = registry
+    end
+
+    def exec
+      #:nocov:
+      raise NotImplementedError.new
+      #:nocov:
+    end
+  end
+end

data/lib/threatinator/actions/list.rb

@@ -0,0 +1,2 @@
+require 'threatinator/actions/list/action'
+require 'threatinator/actions/list/config'

data/lib/threatinator/actions/list/action.rb

@@ -0,0 +1,53 @@
+require 'threatinator/action'
+require 'threatinator/exceptions'
+require 'csv'
+
+module Threatinator
+  module Actions
+    module List
+      class Action < Threatinator::Action
+        def initialize(registry, config)
+          super(registry)
+          @config = config
+        end
+
+        def exec
+          io_out = $stdout
+          feed_info = [['provider', 'name', 'type', 'link/path']]
+          registry.each do |feed|
+            info = [ feed.provider, feed.name ]
+            fetcher = feed.fetcher_builder.call()
+            type = "unknown"
+            link = "unknown"
+            case fetcher
+            when Threatinator::Fetchers::Http
+              type = "http"
+              link = fetcher.url
+            end
+            info << type
+            info << link
+            feed_info << info
+          end
+          return if feed_info.count == 0
+          fmts = []
+          widths = []
+          0.upto(3) do |i|
+            max = feed_info.max { |a,b| a[i].length <=> b[i].length }[i].length
+            widths << max
+            fmts << "%#{max}s"
+          end
+          fmt = "%-#{widths[0]}s  %-#{widths[1]}s  %-#{widths[2]}s  %-#{widths[3]}s\n"
+          io_out.printf(fmt, *(feed_info.shift))
+          sep = widths.map {|x| '-' * x }
+          io_out.printf(fmt, *sep)
+          feed_info.sort! { |a,b| [a[0], a[1]] <=> [b[0], b[1]] }
+          feed_info.each  do |info|
+            io_out.printf(fmt, *info)
+          end
+          io_out.printf(fmt, *sep)
+          io_out.puts("Total: #{feed_info.count}")
+        end
+      end
+    end
+  end
+end

data/lib/threatinator/actions/list/config.rb

@@ -0,0 +1,10 @@
+require 'threatinator/config/base'
+
+module Threatinator
+  module Actions
+    module List
+      class Config < Threatinator::Config::Base
+      end
+    end
+  end
+end

data/lib/threatinator/actions/run.rb

@@ -0,0 +1,2 @@
+require 'threatinator/actions/run/action'
+require 'threatinator/actions/run/config'

data/lib/threatinator/actions/run/action.rb

@@ -0,0 +1,45 @@
+require 'threatinator/action'
+require 'threatinator/exceptions'
+require 'threatinator/feed_runner'
+require 'csv'
+
+module Threatinator
+  module Actions
+    module Run
+      class Action < Threatinator::Action
+        def initialize(registry, config)
+          super(registry)
+          @config = config
+        end
+
+        def build_output
+          @config.output.build_output
+        end
+
+        def exec
+          opts = {}
+
+          feed = registry.get(@config.feed_provider, @config.feed_name)
+          if feed.nil?
+            raise Threatinator::Exceptions::UnknownFeed.new(@config.feed_provider, @config.feed_name)
+          end
+
+          output = build_output
+
+          feed_runner = Threatinator::FeedRunner.new(feed, output)
+
+          @config.observers.each do |observer|
+            feed_runner.add_observer(observer)
+          end
+
+          feed_runner.run
+
+#          if feed_report.num_records_missed != 0
+#            $stderr.puts "WARNING: #{feed_report.num_records_missed} lines/records were MISSED (neither filtered nor parsed). You may need to update your feed specification! Rerun with --coverage to see which records are parsed/filtered/missed" 
+#          end
+
+        end
+      end
+    end
+  end
+end

data/lib/threatinator/actions/run/config.rb

@@ -0,0 +1,32 @@
+require 'threatinator/config/base'
+require 'threatinator/actions/run/output_config'
+
+module Threatinator
+  module Actions
+    module Run
+      module Config
+        # @param [Threatinator::PluginLoader] plugin_loader
+        # @return [Class] a class that represents the config for Action::Run
+        def self.generate(plugin_loader)
+          output_config_class = Threatinator::Actions::Run::OutputConfig.generate(plugin_loader)
+          config_class = Class.new(Threatinator::Config::Base) do
+            attribute :output, output_config_class, 
+              default: lambda { |c,a| output_config_class.new }
+
+            attribute :feed_provider, String, 
+              description: "The feed provider" 
+
+            attribute :feed_name, String, 
+              description: "The feed name" 
+
+            attribute :fetch_from_file, String, 
+              description: "Read data from the specified file rather than fetching"
+
+            attribute :observers, Array, default:  lambda {|c,a| Array.new }
+          end
+          config_class
+        end
+      end
+    end
+  end
+end

data/lib/threatinator/actions/run/coverage_observer.rb

@@ -0,0 +1,54 @@
+require 'csv'
+
+module Threatinator
+  module Actions
+    module Run
+      class CoverageObserver
+        attr_reader :filename
+        def initialize(filename)
+          @filename = filename
+          @csv = nil
+        end
+
+        def closed?
+          return false if @csv.nil?
+          @csv.closed?
+        end
+
+        def open
+          @csv = ::CSV.open(@filename, "wb", :headers => [:status, :event_count, :line_number, :pos_start, :pos_end, :data], :write_headers => true)
+        end
+
+        # Handles FeedRunner observations
+        def update(message, *args)
+          case message
+          when :record_missed
+            log_record(:missed, args.shift, [])
+          when :record_filtered
+            log_record(:filtered, args.shift, [])
+          when :record_parsed
+            log_record(:parsed, args.shift, args.shift)
+          when :end
+            close
+          when :start
+            open
+          end
+        end
+
+        # @param [Symbol] status :parsed, :missed, :filtered
+        # @param [Threatinator::Record] record
+        # @param [Array<Threatinator::Event>] events
+        def log_record(status, record, events = [])
+          return if closed?
+          @csv.add_row(  [
+            status, events.count, record.line_number, 
+            record.pos_start, record.pos_end, record.data.inspect])
+        end
+
+        def close
+          @csv.close unless closed?
+        end
+      end
+    end
+  end
+end

data/lib/threatinator/actions/run/output_config.rb

@@ -0,0 +1,59 @@
+require 'threatinator/config/base'
+require 'threatinator/exceptions'
+
+module Threatinator
+  module Actions
+    module Run
+      module OutputConfigClassMethods
+        def set_plugin_loader(pl)
+          @plugin_loader = pl
+          pl.each(:output) do |type, name, plugin|
+            self.attribute name, plugin::Config, default: lambda { |c,a| plugin::Config.new }
+          end
+        end
+
+        def get_plugin(name)
+          @plugin_loader.get(:output, name)
+        end
+
+        def formats
+          @plugin_loader.each(:output).map { |t, k, p| k.to_s }
+        end
+
+        def formats_str
+          formats.sort.join(', ')
+        end
+
+      end
+
+      module OutputConfigMethods
+        def build_output
+          oc = self.class.get_plugin(self.format)
+          if oc.nil?
+            raise Threatinator::Exceptions::UnknownPlugin.new("Unknown output plugin: '#{format}'")
+          end
+          output_config = self[format]
+
+          if output_config.nil?
+            raise Threatinator::Exceptions::CouldNotFindOutputConfigError.new("Could not find output config for '#{format}'. Perhaps there's some load-order issues?")
+          end
+
+          oc.new(output_config)
+        end
+      end
+
+      module OutputConfig
+        def self.generate(plugin_loader)
+          anonymous_class = Class.new(Threatinator::Config::Base) do
+            extend OutputConfigClassMethods
+            include OutputConfigMethods
+            set_plugin_loader(plugin_loader)
+            attribute :format, Symbol, default: lambda { |c,a| :csv },
+              description: lambda { |c, a| "Output format (#{c.formats_str})" }
+          end
+          anonymous_class
+        end
+      end
+    end
+  end
+end

data/lib/threatinator/cli.rb

@@ -0,0 +1,13 @@
+require 'threatinator/cli/parser'
+module Threatinator
+  module CLI
+    def self.process!(args)
+      parser = Parser.new
+      ret = parser.parse(args)
+      if builder = parser.builder
+        builder.build.exec
+      end
+      ret
+    end
+  end
+end

data/lib/threatinator/cli/action_builder.rb

@@ -0,0 +1,33 @@
+require 'threatinator/config'
+require 'threatinator/feed_registry'
+
+module Threatinator
+  module CLI
+    class ActionBuilder
+      attr_reader :extra_args, :config_hash
+
+      def initialize(config_hash, extra_args)
+        @extra_args = extra_args
+        @config_hash = config_hash
+        @feed_registry = nil
+      end
+
+      def build
+        #:nocov:
+        raise NotImplementedError.new("#{self.class}#build not implemented")
+        #:nocov:
+      end
+
+      def feed_registry
+        return @feed_registry unless @feed_registry.nil?
+
+        feed_search_hash = config_hash["feed_search"] || {}
+        feed_search_config = Threatinator::Config::FeedSearch.new(feed_search_hash)
+
+        @feed_registry = Threatinator::FeedRegistry.build(feed_search_config)
+      end
+    end
+
+  end
+end
+

data/lib/threatinator/cli/list_action_builder.rb

@@ -0,0 +1,19 @@
+require 'threatinator/cli/action_builder'
+require 'threatinator/actions/list'
+
+module Threatinator
+  module CLI
+    class ListActionBuilder < ActionBuilder
+      def build
+        Threatinator::Actions::List::Action.new(feed_registry, config)
+      end
+
+      def config
+        list_hash = config_hash["list"] || {}
+        Threatinator::Actions::List::Config.new(list_hash)
+      end
+
+    end
+  end
+end
+

data/lib/threatinator/cli/parser.rb

@@ -0,0 +1,113 @@
+require 'gli'
+require 'threatinator/actions/run'
+require 'threatinator/actions/list'
+require 'threatinator/config/feed_search'
+require 'threatinator/cli/list_action_builder'
+require 'threatinator/cli/run_action_builder'
+require 'threatinator/plugin_loader'
+
+module Threatinator
+  module CLI
+    module Helpers
+      def add_cli_args(gli, config_properties)
+        config_properties.each do |key, args|
+          desc, type, default_value = args
+          if type.base == Axiom::Types::Boolean
+            gli.switch key, desc: desc
+          elsif type.base ==Axiom::Types::Array
+            gli.flag key, desc: desc, type: Array
+          else 
+            gli.flag key, desc: desc
+          end
+        end
+      end
+
+      def clean_opts(opts)
+        opts.delete_if {|k, v| k.kind_of?(::Symbol) or (k == 'help') or v.nil? }
+        opts
+      end
+
+      def nest_opts(opts)
+        config_hash = {}
+        opts.each_pair do |key, val|
+          key_path = key.to_s.split('.')
+          final_key = key_path.pop
+          nested_hash = config_hash
+          while key_path.length > 0
+            part = key_path.shift
+            nested_hash[part] ||= {}
+            nested_hash = nested_hash[part]
+          end
+          nested_hash[final_key] = val
+        end
+        config_hash
+      end
+
+      def fix_opts(opts)
+        nest_opts(clean_opts(opts))
+      end
+    end
+
+    class Parser
+      attr_accessor :builder
+      attr_reader :run_action_config_class
+
+      def initialize
+        @builder = nil
+        @plugin_loader = Threatinator::PluginLoader.new
+        @plugin_loader.load_all_plugins
+        @run_action_config_class = Threatinator::Actions::Run::Config.generate(@plugin_loader)
+        @mod = _init_mod
+      end
+
+      def parse(args)
+        @mod.run(args)
+      end
+
+      def _init_mod
+        parser = self
+        Module.new do
+          extend Helpers
+          extend GLI::App
+
+          program_desc 'Threatinator!'
+
+          add_cli_args(self, Threatinator::Config::FeedSearch.properties('feed_search'))
+
+          desc "fetch and parse a feed"
+          command :run do |c|
+            c.flag 'run.coverage_output', type: String, desc: "Write coverage analysis to the specified file (CSV format)"
+
+            add_cli_args(c, parser.run_action_config_class.properties('run'))
+            c.action do |global_options, options, args|
+              if options["run.feed_provider"].nil?
+                options["run.feed_provider"] = args.shift
+              end
+
+              if options["run.feed_name"].nil?
+                options["run.feed_name"] = args.shift
+              end
+
+              opts = fix_opts(global_options.merge(options))
+
+              builder = Threatinator::CLI::RunActionBuilder.new(opts, args, parser.run_action_config_class)
+              parser.builder = builder
+            end
+          end
+
+          desc 'list out all the feeds'
+          command :list do |c|
+            add_cli_args(c, Threatinator::Actions::List::Config.properties('list'))
+            c.action do |global_options, options, args|
+              opts = fix_opts(global_options.merge(options))
+              parser.builder = Threatinator::CLI::ListActionBuilder.new(opts, args)
+            end
+          end
+        end
+      end
+
+    end # _init_mod
+  end
+
+end
+