threatinator 0.1.1

data/lib/threatinator/parsers/xml/pattern.rb

@@ -0,0 +1,53 @@
+require 'threatinator/parsers/xml/path'
+
+module Threatinator
+  module Parsers
+    module XML
+      # Implements path matching behavior for use with the XML parser. Aims to support
+      # a small subset of XPath behaviors, specifically for matching elements. 
+      class Pattern
+        # @param [String] pathspec A specification of a path match.
+        def initialize(pathspec)
+          parts = pathspec.split('/')
+          leader_count = 0
+          while parts[0] == ''
+            leader_count += 1
+            parts.shift
+          end
+          @path = Threatinator::Parsers::XML::Path.new(parts)
+          @anchored = true
+
+          if leader_count == 1
+            @anchored = true
+          elsif leader_count == 2
+            @anchored = false
+          else
+            raise ArgumentError.new('pathspec must begin with "/" or "//"')
+          end
+        end
+
+        def max_depth
+          @anchored == true ? @path.length : Float::INFINITY
+        end
+
+        # @param [Threatinator::Parsers::XML::Path] path 
+        # @return [Boolean] true if the pattern matches, false otherwise.
+        def match?(path)
+          if @anchored == true
+            @path == path
+          else 
+            path.end_with?(@path)
+          end
+        end
+
+        def _internal_data
+          [@path, @anchored]
+        end
+
+        def ==(other)
+          _internal_data == other._internal_data
+        end
+      end
+    end
+  end
+end

data/lib/threatinator/parsers/xml/record.rb

@@ -0,0 +1,14 @@
+require 'threatinator/record'
+
+module Threatinator
+  module Parsers
+    module XML
+      class Record < Threatinator::Record
+        alias_method :node, :data
+        def initialize(node, opts = {})
+          super(node, opts)
+        end
+      end
+    end
+  end
+end

data/lib/threatinator/parsers/xml/sax_document.rb

@@ -0,0 +1,64 @@
+require 'threatinator/parsers/xml/node_builder'
+require 'threatinator/parsers/xml/path'
+require 'nokogiri'
+
+module Threatinator
+  module Parsers
+    module XML
+      class SAXDocument < Nokogiri::XML::SAX::Document
+        def initialize(pattern, cb)
+          @pattern = pattern
+          @max_depth = @pattern.max_depth
+          @cb = cb
+          @element_stack = Threatinator::Parsers::XML::Path.new
+          @parsing_stack = []
+          @current_node = nil
+          super()
+        end
+
+        def start_parsing(name, attributes)
+          @current_node = Threatinator::Parsers::XML::NodeBuilder.new(name, attributes)
+          @parsing_stack.push(@current_node)
+        end
+
+        def characters(str)
+          return if @current_node.nil?
+          @current_node.append_text(str)
+        end
+
+        alias_method :cdata_block, :characters
+
+        def start_element_namespace(name, attrs = [], prefix = nil, uri = nil, ns = [])
+          @element_stack.push(name)
+
+          if @parsing_stack.empty?
+            if @element_stack.length > @max_depth
+              return
+            end
+
+            if @pattern.match?(@element_stack)
+              start_parsing(name, attrs)
+            end
+          else
+            start_parsing(name, attrs)
+          end
+        end
+
+        def end_element_namespace(name, prefix = nil, uri = nil)
+          name_sym = name.to_sym
+          @element_stack.pop
+          return if @parsing_stack.empty?
+          @parsing_stack.pop
+
+          if parent = @parsing_stack.last
+            parent.add_child(@current_node.build)
+            @current_node = parent
+          else
+            @cb.call(@current_node.build)
+            @current_node = nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/threatinator/plugin_loader.rb

@@ -0,0 +1,115 @@
+require 'rubygems'
+require 'threatinator/util'
+require 'threatinator/registry'
+
+module Threatinator
+  class PluginLoader
+    def initialize()
+      @plugin_types_registry = Threatinator::Registry.new
+    end
+
+    # Loads all plugins
+    # @return [Threatinator::PluginLoader] self
+    def load_all_plugins()
+      load_plugins(:*)
+      return self
+    end
+
+    # Loads all plugins of the specified type
+    # @param [#to_sym] type
+    # @return [Threatinator::PluginLoader] self
+    def load_plugins(type)
+      load_files(find_plugin_files(type.to_sym))
+      return self
+    end
+
+    # Retrieves a loaded plugin by type and name.
+    # @param [#to_sym] type
+    # @param [#to_sym] name
+    # @return [Object] plugin
+    def get(type, name)
+      type_registry = @plugin_types_registry.get(type.to_sym)
+      return nil if type_registry.nil?
+      return type_registry.get(name.to_sym)
+    end
+
+    # Enumerates through all plugins, optionally filtered by type.
+    # @yield [type, name, plugin]
+    # @yieldparam [Symbol] type
+    # @yieldparam [Symbol] name
+    # @yieldparam [Object] plugin
+    def each(type = nil)
+      return enum_for(:each, type) unless block_given?
+      @plugin_types_registry.each do |t, type_registry|
+        unless type.nil?
+          next unless type.to_sym == t
+        end
+        type_registry.each do |name, plugin|
+          yield(t, name, plugin)
+        end
+      end
+    end
+
+    # Returns an array of all the types of plugins that are loaded.
+    # @return [Array<Symbol>]
+    def types
+      @plugin_types_registry.keys
+    end
+
+    # Registers a plugin with the provided type and name.
+    # @param [#to_sym] type
+    # @param [#to_sym] name
+    # @param [Object] plugin
+    # @raise [Threatinator::Exceptions::AlreadyRegisteredError
+    def register_plugin(type, name, plugin)
+      type = type.to_sym
+      unless type_registry = @plugin_types_registry.get(type)
+        type_registry = @plugin_types_registry.register(type, Threatinator::Registry.new)
+      end
+      type_registry.register(name.to_sym, plugin)
+    end
+
+    private
+
+    def find_plugin_files(type)
+      Gem.find_files("threatinator/plugins/#{type}/*.rb")
+    end
+
+    def register_plugin_by_name(type, name)
+      plugin_name = "Threatinator::Plugins::#{Threatinator::Util.underscore2cc(type)}::#{Threatinator::Util.underscore2cc(name)}"
+      begin
+        type_obj = Threatinator::Plugins.const_get(Threatinator::Util.underscore2cc(type))
+        plugin = type_obj.const_get(Threatinator::Util.underscore2cc(name))
+        register_plugin(type, name, plugin)
+      rescue ::NameError => e
+        raise Threatinator::Exceptions::PluginLoadError.new("Failed to load plugin '#{plugin_name}'", e)
+      end
+    end
+
+    def load_files(file_names)
+      file_names.each do |file_name|
+        path, type, name = split_file_name(file_name)
+        next if path.nil?
+        # Don't try to load unit tests as plugins :)
+        next if name.end_with?("_spec")
+        begin 
+          require path
+        rescue ::LoadError
+          # TODO: Handle plugins inside of gems a bit better. We should try 
+          #  using only the latest version of a given Gem.
+          next
+        end
+
+        register_plugin_by_name(type, name)
+      end
+    end
+
+    # @return [Array, nil] an array containing the path, type, and name of the
+    #  plugin
+    def split_file_name(file_name)
+      m = file_name.match(/(?<=(\A|\/))(?<path>threatinator\/plugins\/(?<type>[^\/]+)\/(?<name>[^\/\.]+))\.rb$/)
+      return nil if m.nil?
+      [m[:path], m[:type], m[:name]]
+    end
+  end
+end

data/lib/threatinator/plugins/output/csv.rb

@@ -0,0 +1,47 @@
+require 'threatinator/output'
+require 'csv'
+module Threatinator
+  module Plugins
+    module Output
+      class Csv < Threatinator::FileBasedOutput
+        class Config < superclass::Config
+        end
+
+        def initialize(config)
+          super(config)
+          @csv = ::CSV.new(self.output_io, 
+                           :write_headers => true,
+                           :headers => [
+                             :provider,
+                             :feed_name,
+                             :type,
+                             :ipv4_1,
+                             :ipv4_2,
+                             :ipv4_3,
+                             :ipv4_4,
+                             :fqdn_1,
+                             :fqdn_2,
+                             :fqdn_3,
+                             :fqdn_4
+                           ])
+        end
+
+        def handle_event(event)
+          @csv.add_row([
+            event.feed_provider,
+            event.feed_name,
+            event.type,
+            event.ipv4s[0],
+            event.ipv4s[1],
+            event.ipv4s[2],
+            event.ipv4s[3],
+            event.fqdns[0],
+            event.fqdns[1],
+            event.fqdns[2],
+            event.fqdns[3]
+          ])
+        end
+      end
+    end
+  end
+end

data/lib/threatinator/plugins/output/null.rb

@@ -0,0 +1,17 @@
+require 'threatinator/output'
+module Threatinator
+  module Plugins
+    module Output
+      class Null < Threatinator::Output
+        class Config < superclass::Config
+        end
+
+        def handle_event(event)
+        end
+
+        def finish
+        end
+      end
+    end
+  end
+end

data/lib/threatinator/plugins/output/rubydebug.rb

@@ -0,0 +1,16 @@
+require 'threatinator/output'
+require 'pp'
+module Threatinator
+  module Plugins
+    module Output
+      class Rubydebug < Threatinator::FileBasedOutput
+        class Config < superclass::Config
+        end
+
+        def handle_event(event)
+          ::PP.pp(event, self.output_io); nil
+        end
+      end
+    end
+  end
+end

data/lib/threatinator/property_definer.rb

@@ -0,0 +1,101 @@
+
+module Threatinator
+  # A helper that lets us easily define properties within an object and
+  # validate them.
+  module PropertyDefiner
+    def self.included(base)
+      base.extend(Threatinator::PropertyDefiner::ClassMethods)
+    end
+
+    def _properties
+      @_properties ||= Hash.new
+    end
+
+    def _prop(name)
+      self.class._properties[name]
+    end
+
+    def _get(name)
+      if _properties.has_key?(name)
+        return _properties[name]
+      end
+      prop = _prop(name)
+      return nil if prop.nil?
+      val = prop.default_value
+      _set(name, val)
+      val
+    end
+
+    def _set(name, val)
+      prop = _prop(name)
+      return if prop.nil?
+      prop.validate!(self, val)
+      _properties[name] = val
+    end
+
+    def _parse_properties(opts = {})
+      opts.each { |k, v| _set(k, v) }
+    end
+
+    class Property
+      attr_reader :name, :opts
+      def initialize(name, opts = {})
+        @name = name.to_sym
+        @type = opts.delete(:type) || Object
+        if @validator = opts.delete(:validate) 
+          unless @validator.kind_of?(Proc)
+            raise ArgumentError.new(":validate must be a proc")
+          end
+        end
+        @default_value = opts.delete(:default)
+        @opts = opts
+      end
+
+      def default_value
+        case @default_value
+        when Proc
+          return @default_value.call()
+        when nil
+          return nil
+        else 
+          return @default_value
+        end
+      end
+
+      def valid?(obj, val)
+        return false unless val.kind_of?(@type)
+        unless @validator.nil?
+          return false unless @validator.call(obj, val)
+        end
+        true
+      end
+
+      def validate!(obj, val)
+        unless valid?(obj, val)
+          raise Threatinator::Exceptions::InvalidAttributeError.new(self.name, val)
+        end
+      end
+    end
+
+    module ClassMethods
+      def _properties
+        @@properties ||= {}
+      end
+
+      def _define_property(prop)
+        name = prop.name
+        _properties[name] = prop
+        define_method("#{prop.name}=".to_sym) do |newval|
+          _set(name, newval)
+        end
+        define_method(name) do 
+          _get(name)
+        end
+      end
+
+      def property(name, opts = {})
+        _define_property(Property.new(name, opts))
+      end
+    end
+  end
+end

data/lib/threatinator/record.rb

@@ -0,0 +1,22 @@
+module Threatinator
+  class Record
+    attr_reader :data, :line_number, :pos_start, :pos_end
+    def initialize(data, opts = {})
+      @data = data
+      @line_number = opts[:line_number]
+      @pos_start = opts[:pos_start]
+      @pos_end = opts[:pos_end]
+    end
+
+    def ==(other)
+      @data == other.data &&
+        @line_number == other.line_number &&
+        @pos_start == other.pos_start &&
+        @pos_end == other.pos_end
+    end
+
+    def eql?(other)
+      other.kind_of?(self.class) && self == other
+    end
+  end
+end