shadowbq-threatinator 0.5.0

data/lib/threatinator/parsers/xml/node_builder.rb

@@ -0,0 +1,39 @@
+require 'threatinator/parsers/xml/node'
+
+module Threatinator
+  module Parsers
+    module XML
+      class NodeBuilder
+        def initialize(name, attributes)
+          @name = name
+          @attributes = {}
+          @children = []
+          @text = ""
+
+          unless attributes.empty?
+            attributes.each { |attr| self.add_attribute(attr.localname, attr.value) }
+          end
+        end
+
+        def append_text(chars)
+          @text << chars
+        end
+
+        def add_attribute(name, value)
+          @attributes[name.to_sym] = value
+        end
+
+        def add_child(node)
+          @children << node
+        end
+
+        def build
+          Threatinator::Parsers::XML::Node.new(@name, 
+                                               attrs: @attributes, 
+                                               text: @text.strip, 
+                                               children: @children)
+        end
+      end
+    end
+  end
+end

data/lib/threatinator/parsers/xml/parser.rb

@@ -0,0 +1,44 @@
+require 'threatinator/parser'
+require 'nokogiri'
+
+module Threatinator
+  module Parsers
+    module XML
+      class Parser < Threatinator::Parser
+        require 'threatinator/parsers/xml/path'
+        require 'threatinator/parsers/xml/record'
+        require 'threatinator/parsers/xml/sax_document'
+
+        attr_reader :pattern
+        # @param [Hash] opts Parameters hash
+        # @option opts [Threatinator::Parsers::XML::Pattern] :pattern The pattern
+        #  object to use for matching chunks of XML
+        def initialize(opts = {})
+          @pattern = opts.delete(:pattern) or raise ArgumentError.new("Missing argument :pattern")
+          @max_cursor_depth = @pattern.max_depth - 1
+          super(opts)
+        end
+
+        def ==(other)
+          @pattern == other.pattern && 
+            super(other)
+        end
+
+        # @param [IO] io
+        # @yield [record] Gives one line to the block
+        # @yieldparam record [Threatinator::Parser::XML::Record] a record
+        def run(io)
+          stack = Path.new
+          callback = lambda do |element|
+            yield(Threatinator::Parsers::XML::Record.new(element))
+          end
+
+          doc = SAXDocument.new(@pattern, callback)
+          parser = Nokogiri::XML::SAX::Parser.new(doc)
+          parser.parse(io)
+        end
+
+      end
+    end
+  end
+end

data/lib/threatinator/parsers/xml/path.rb

@@ -0,0 +1,70 @@
+module Threatinator
+  module Parsers
+    module XML
+      class Path
+        attr_reader :parts
+
+        # @param [String, Array, nil] str_or_parts ([]) If set to a String, splits 
+        #   the string by '/' into an array. If set to an Array, sets parts to a
+        #   duplicate of the array. If set to nil or not specified, defaults to 
+        #   a new array.
+        # @raise [TypeError] if something other than a String, Array, or nil is 
+        #   specified for str_or_parts.
+        def initialize(str_or_parts = nil)
+          @parts = 
+            case str_or_parts
+            when ::String
+              if str_or_parts.length == 0 or !str_or_parts.start_with?('/')
+                raise ArgumentError.new('str_or_parts must be a String beginning with "/"')
+              end
+              r = str_or_parts.split('/')
+              r.shift
+              r
+            when ::Array
+              str_or_parts.dup
+            when nil
+              []
+            else
+              raise TypeError.new("Expected argument must be a String, Array, or nil")
+            end
+        end
+
+        def ==(other)
+          @parts == other.parts
+        end
+
+        def eql?(other)
+          other.kind_of?(self.class) &&
+            self == other
+        end
+
+        # length = 5
+        #   0 1 2 3 4
+        #  /a/b/c/d/e
+        #         0 1
+        #        /d/e
+        def end_with?(other_path)
+          return false if other_path.length > self.length
+          return true if other_path.length == 0
+          pos = length - other_path.length
+          other_path.parts.each_with_index do |other_part, idx|
+            return false unless @parts[(pos + idx)] == other_part
+          end
+          true
+        end
+
+        def push(name)
+          @parts.push(name)
+        end
+
+        def pop
+          @parts.pop
+        end
+
+        def length
+          @parts.length
+        end
+      end
+    end
+  end
+end

data/lib/threatinator/parsers/xml/pattern.rb

@@ -0,0 +1,53 @@
+require 'threatinator/parsers/xml/path'
+
+module Threatinator
+  module Parsers
+    module XML
+      # Implements path matching behavior for use with the XML parser. Aims to support
+      # a small subset of XPath behaviors, specifically for matching elements. 
+      class Pattern
+        # @param [String] pathspec A specification of a path match.
+        def initialize(pathspec)
+          parts = pathspec.split('/')
+          leader_count = 0
+          while parts[0] == ''
+            leader_count += 1
+            parts.shift
+          end
+          @path = Threatinator::Parsers::XML::Path.new(parts)
+          @anchored = true
+
+          if leader_count == 1
+            @anchored = true
+          elsif leader_count == 2
+            @anchored = false
+          else
+            raise ArgumentError.new('pathspec must begin with "/" or "//"')
+          end
+        end
+
+        def max_depth
+          @anchored == true ? @path.length : Float::INFINITY
+        end
+
+        # @param [Threatinator::Parsers::XML::Path] path 
+        # @return [Boolean] true if the pattern matches, false otherwise.
+        def match?(path)
+          if @anchored == true
+            @path == path
+          else 
+            path.end_with?(@path)
+          end
+        end
+
+        def _internal_data
+          [@path, @anchored]
+        end
+
+        def ==(other)
+          _internal_data == other._internal_data
+        end
+      end
+    end
+  end
+end

data/lib/threatinator/parsers/xml/record.rb

@@ -0,0 +1,14 @@
+require 'threatinator/record'
+
+module Threatinator
+  module Parsers
+    module XML
+      class Record < Threatinator::Record
+        alias_method :node, :data
+        def initialize(node, opts = {})
+          super(node, opts)
+        end
+      end
+    end
+  end
+end

data/lib/threatinator/parsers/xml/sax_document.rb

@@ -0,0 +1,64 @@
+require 'threatinator/parsers/xml/node_builder'
+require 'threatinator/parsers/xml/path'
+require 'nokogiri'
+
+module Threatinator
+  module Parsers
+    module XML
+      class SAXDocument < Nokogiri::XML::SAX::Document
+        def initialize(pattern, cb)
+          @pattern = pattern
+          @max_depth = @pattern.max_depth
+          @cb = cb
+          @element_stack = Threatinator::Parsers::XML::Path.new
+          @parsing_stack = []
+          @current_node = nil
+          super()
+        end
+
+        def start_parsing(name, attributes)
+          @current_node = Threatinator::Parsers::XML::NodeBuilder.new(name, attributes)
+          @parsing_stack.push(@current_node)
+        end
+
+        def characters(str)
+          return if @current_node.nil?
+          @current_node.append_text(str)
+        end
+
+        alias_method :cdata_block, :characters
+
+        def start_element_namespace(name, attrs = [], prefix = nil, uri = nil, ns = [])
+          @element_stack.push(name)
+
+          if @parsing_stack.empty?
+            if @element_stack.length > @max_depth
+              return
+            end
+
+            if @pattern.match?(@element_stack)
+              start_parsing(name, attrs)
+            end
+          else
+            start_parsing(name, attrs)
+          end
+        end
+
+        def end_element_namespace(name, prefix = nil, uri = nil)
+          name_sym = name.to_sym
+          @element_stack.pop
+          return if @parsing_stack.empty?
+          @parsing_stack.pop
+
+          if parent = @parsing_stack.last
+            parent.add_child(@current_node.build)
+            @current_node = parent
+          else
+            @cb.call(@current_node.build)
+            @current_node = nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/threatinator/parsers/xml.rb

@@ -0,0 +1,8 @@
+module Threatinator
+  module Parsers
+    module XML
+      require 'threatinator/parsers/xml/parser'
+      require 'threatinator/parsers/xml/pattern'
+    end
+  end
+end

data/lib/threatinator/plugin_loader.rb

@@ -0,0 +1,115 @@
+require 'rubygems'
+require 'threatinator/util'
+require 'threatinator/registry'
+
+module Threatinator
+  class PluginLoader
+    def initialize()
+      @plugin_types_registry = Threatinator::Registry.new
+    end
+
+    # Loads all plugins
+    # @return [Threatinator::PluginLoader] self
+    def load_all_plugins()
+      load_plugins(:*)
+      return self
+    end
+
+    # Loads all plugins of the specified type
+    # @param [#to_sym] type
+    # @return [Threatinator::PluginLoader] self
+    def load_plugins(type)
+      load_files(find_plugin_files(type.to_sym))
+      return self
+    end
+
+    # Retrieves a loaded plugin by type and name.
+    # @param [#to_sym] type
+    # @param [#to_sym] name
+    # @return [Object] plugin
+    def get(type, name)
+      type_registry = @plugin_types_registry.get(type.to_sym)
+      return nil if type_registry.nil?
+      return type_registry.get(name.to_sym)
+    end
+
+    # Enumerates through all plugins, optionally filtered by type.
+    # @yield [type, name, plugin]
+    # @yieldparam [Symbol] type
+    # @yieldparam [Symbol] name
+    # @yieldparam [Object] plugin
+    def each(type = nil)
+      return enum_for(:each, type) unless block_given?
+      @plugin_types_registry.each do |t, type_registry|
+        unless type.nil?
+          next unless type.to_sym == t
+        end
+        type_registry.each do |name, plugin|
+          yield(t, name, plugin)
+        end
+      end
+    end
+
+    # Returns an array of all the types of plugins that are loaded.
+    # @return [Array<Symbol>]
+    def types
+      @plugin_types_registry.keys
+    end
+
+    # Registers a plugin with the provided type and name.
+    # @param [#to_sym] type
+    # @param [#to_sym] name
+    # @param [Object] plugin
+    # @raise [Threatinator::Exceptions::AlreadyRegisteredError
+    def register_plugin(type, name, plugin)
+      type = type.to_sym
+      unless type_registry = @plugin_types_registry.get(type)
+        type_registry = @plugin_types_registry.register(type, Threatinator::Registry.new)
+      end
+      type_registry.register(name.to_sym, plugin)
+    end
+
+    private
+
+    def find_plugin_files(type)
+      Gem.find_files("threatinator/plugins/#{type}/*.rb")
+    end
+
+    def register_plugin_by_name(type, name)
+      plugin_name = "Threatinator::Plugins::#{Threatinator::Util.underscore2cc(type)}::#{Threatinator::Util.underscore2cc(name)}"
+      begin
+        type_obj = Threatinator::Plugins.const_get(Threatinator::Util.underscore2cc(type))
+        plugin = type_obj.const_get(Threatinator::Util.underscore2cc(name))
+        register_plugin(type, name, plugin)
+      rescue ::NameError => e
+        raise Threatinator::Exceptions::PluginLoadError.new("Failed to load plugin '#{plugin_name}'", e)
+      end
+    end
+
+    def load_files(file_names)
+      file_names.each do |file_name|
+        path, type, name = split_file_name(file_name)
+        next if path.nil?
+        # Don't try to load unit tests as plugins :)
+        next if name.end_with?("_spec")
+        begin 
+          require path
+        rescue ::LoadError
+          # TODO: Handle plugins inside of gems a bit better. We should try 
+          #  using only the latest version of a given Gem.
+          next
+        end
+
+        register_plugin_by_name(type, name)
+      end
+    end
+
+    # @return [Array, nil] an array containing the path, type, and name of the
+    #  plugin
+    def split_file_name(file_name)
+      m = file_name.match(/(?<=(\A|\/))(?<path>threatinator\/plugins\/(?<type>[^\/]+)\/(?<name>[^\/\.]+))\.rb$/)
+      return nil if m.nil?
+      [m[:path], m[:type], m[:name]]
+    end
+  end
+end

data/lib/threatinator/plugins/output/amqp/config.rb

@@ -0,0 +1,18 @@
+require 'threatinator/output'
+
+module Threatinator
+  module Plugins
+    module Output
+      require 'threatinator/plugins/output/amqp'
+      class Amqp
+        class Config < Threatinator::Output::Config
+          attribute :url, String, description: "The hostname/ip of the RabbitMQ server"
+
+          attribute :routing_key, String, default: lambda { |c,a| 'amqp.event' },
+            description: "Routing key for Amqp events"
+
+        end
+      end
+    end
+  end
+end

data/lib/threatinator/plugins/output/amqp.rb

@@ -0,0 +1,41 @@
+require 'threatinator/output'
+require 'bunny'
+require 'multi_json'
+require 'pry'
+# bundle exec threatinator run --run.output.amqp.url=#{ENV['RABBITMQ_URL']} --run.output.format=amqp #{provider} #{name}"
+#    --run.output.amqp.routing_key=arg   - Routing key for Amqp events (default: none)
+#    --run.output.amqp.url=arg           - The hostname/ip of the RabbitMQ server (default: none)
+
+
+module Threatinator
+  module Plugins
+    module Output
+      class Amqp < Threatinator::Output
+        require 'threatinator/plugins/output/amqp/config'
+
+        def initialize(config)
+          bunny_config = {
+            recover_from_connection_close: true,
+            network_recovery_interval: 5.0
+          }
+          @bunny = Bunny.new(config.url, bunny_config)
+          @bunny.start
+          @channel = @bunny.create_channel
+          @exchange = @channel.topic("threats")
+        end
+
+        def handle_event(event)
+          @routing_key = 'threatinator.' + event.type.to_s
+          @exchange.publish(MultiJson.dump(event.to_serializable_hash),
+                            routing_key: @routing_key)
+        end
+
+        def finish
+          @exchange = nil
+          @bunny.close
+        end
+
+      end
+    end
+  end
+end

data/lib/threatinator/plugins/output/csv.rb

@@ -0,0 +1,58 @@
+require 'threatinator/output'
+require 'csv'
+module Threatinator
+  module Plugins
+    module Output
+      class Csv < Threatinator::FileBasedOutput
+        class Config < superclass::Config
+        end
+
+        def initialize(config)
+          super(config)
+          @csv = ::CSV.new(self.output_io, 
+                           :write_headers => true,
+                           :headers => [
+                             :provider,
+                             :feed_name,
+                             :type,
+                             :ipv4_1,
+                             :ipv4_2,
+                             :ipv4_3,
+                             :ipv4_4,
+                             :fqdn_1,
+                             :fqdn_2,
+                             :fqdn_3,
+                             :fqdn_4,
+                             :url_1,
+                             :url_2,
+                             :url_3,
+                             :url_4
+                           ])
+        end
+
+        def handle_event(event)
+          ipv4s = event.ipv4s.to_a[0..3].map { |o| o.nil? ? nil : o.ipv4.to_addr }
+          fqdns = event.fqdns.to_a[0..3]
+          urls  = event.urls.to_a[0..3].map {|x| x.to_s }
+          @csv.add_row([
+            event.feed_provider,
+            event.feed_name,
+            event.type,
+            ipv4s[0],
+            ipv4s[1],
+            ipv4s[2],
+            ipv4s[3],
+            fqdns[0],
+            fqdns[1],
+            fqdns[2],
+            fqdns[3],
+            urls[0],
+            urls[1],
+            urls[2],
+            urls[3],
+          ])
+        end
+      end
+    end
+  end
+end

data/lib/threatinator/plugins/output/json/config.rb

@@ -0,0 +1,14 @@
+require 'threatinator/output'
+
+module Threatinator
+  module Plugins
+    module Output
+      require 'threatinator/plugins/output/json'
+      class Json
+        class Config < Threatinator::Output::Config
+          attribute :file, String, description: "The hostname/ip of the RabbitMQ server"
+        end
+      end
+    end
+  end
+end

data/lib/threatinator/plugins/output/json.rb

@@ -0,0 +1,53 @@
+require 'threatinator/output'
+require 'multi_json'
+#require 'pry'
+module Threatinator
+  module Plugins
+    module Output
+
+      class Json < Threatinator::Output
+        #include JSONHelpers
+        require 'threatinator/plugins/output/json/config'
+
+        def initialize(config)
+          @output_file = config.file
+          @first = true
+          puts "{"
+        end
+
+        def handle_event(event)
+          if @first
+            puts "\"header\" : #{MultiJson.dump(event.header.to_h)}"
+            puts ",\"items\" : ["
+            puts "{"
+            @first = false
+          else
+            puts ",{"
+          end
+          delim = ""
+          if event.ipv4s.size > 0
+            puts "#{delim}\"ipv4s\" : #{MultiJson.dump(event.ipv4s.list)}"
+            delim = ","
+          end
+          if event.fqdns.size > 0
+            puts "#{delim}\"fqdns\" : #{MultiJson.dump(event.fqdns.list)}"
+            delim = ","
+          end
+          if event.urls.size > 0
+            puts "#{delim}\"urls\" : #{MultiJson.dump(event.urls.list)}"
+            delim = ","
+          end
+          puts "}"
+
+        end
+
+        def finish
+          puts "]}"
+        end
+
+
+
+      end
+    end
+  end
+end

data/lib/threatinator/plugins/output/null.rb

@@ -0,0 +1,17 @@
+require 'threatinator/output'
+module Threatinator
+  module Plugins
+    module Output
+      class Null < Threatinator::Output
+        class Config < superclass::Config
+        end
+
+        def handle_event(event)
+        end
+
+        def finish
+        end
+      end
+    end
+  end
+end

data/lib/threatinator/plugins/output/rubydebug.rb

@@ -0,0 +1,16 @@
+require 'threatinator/output'
+require 'pp'
+module Threatinator
+  module Plugins
+    module Output
+      class Rubydebug < Threatinator::FileBasedOutput
+        class Config < superclass::Config
+        end
+
+        def handle_event(event)
+          ::PP.pp(event, self.output_io); nil
+        end
+      end
+    end
+  end
+end

data/lib/threatinator/record.rb

@@ -0,0 +1,22 @@
+module Threatinator
+  class Record
+    attr_reader :data, :line_number, :pos_start, :pos_end
+    def initialize(data, opts = {})
+      @data = data
+      @line_number = opts[:line_number]
+      @pos_start = opts[:pos_start]
+      @pos_end = opts[:pos_end]
+    end
+
+    def ==(other)
+      @data == other.data &&
+        @line_number == other.line_number &&
+        @pos_start == other.pos_start &&
+        @pos_end == other.pos_end
+    end
+
+    def eql?(other)
+      other.kind_of?(self.class) && self == other
+    end
+  end
+end