shadowbq-threatinator 0.5.0

data/lib/threatinator/model/base.rb

@@ -0,0 +1,23 @@
+require 'active_model'
+require 'active_model/validations'
+require 'threatinator/model/validations'
+require 'threatinator/exceptions'
+
+module Threatinator
+  module Model
+    class Base
+      include ActiveModel::Validations
+      include Threatinator::Model::Validations
+
+      def initialize
+        validate!
+      end
+
+      def validate!
+        unless valid?
+          raise Threatinator::Exceptions::InvalidAttributeError, errors.full_messages.join("\n")
+        end
+      end
+    end
+  end
+end

data/lib/threatinator/model/collection.rb

@@ -0,0 +1,89 @@
+require 'threatinator/exceptions'
+require 'set'
+
+module Threatinator
+  module Model
+    class Collection
+      def initialize(values = [])
+        @collection = Set.new
+        values.each do |v|
+          self << v
+        end
+      end
+
+      def valid_member?(v)
+        #:nocov:
+        raise NotImplementedError, "#valid_member? not implemented"
+        #:nocov:
+      end
+
+      def <<(v)
+        unless valid_member?(v)
+          raise Threatinator::Exceptions::InvalidAttributeError, "Invalid member: #{v.class} '#{v.inspect}'"
+        end
+        @collection << v
+      end
+
+      def include?(member)
+        @collection.include?(member)
+      end
+      alias_method :member?, :include?
+
+      # @return [Boolean] true if empty, false otherwise
+      def empty?
+        @collection.empty?
+      end
+
+      def collect!
+        block_given? or return enum_for(__method__)
+        @collection.replace(@collection.class.new(@collection) { |o| yield(o) })
+      end
+
+      def delete(o)
+        @collection.delete(o)
+      end
+
+      def delete?(o)
+        @collection.delete?(o)
+      end
+
+      # @return [Integer] the number of members in the collection
+      def count
+        @collection.count
+      end
+      alias_method :size, :count
+      alias_method :length, :count
+
+      def to_ary
+        @collection.to_a
+      end
+      alias_method :to_a, :to_ary
+
+      # [31] pry(#<Threatinator::Plugins::Output::Json>)> event.urls.each{ |uri| p uri.to_s }
+      # "http://teamadrenaline.com/js/t1.exe"
+      # =>  [#<Addressable::URI:0x114c6ec URI:http://teamadrenaline.com/js/t1.exe>]
+      def each
+        return to_enum(:each) unless block_given?
+        @collection.each { |v| yield v }
+      end
+
+      def list
+        @collection.to_a.collect {|item|
+          item.to_s
+        }
+      end
+
+      def ==(other)
+        if self.equal?(other)
+          return true
+        elsif other.instance_of?(self.class)
+          @collection == other.instance_variable_get(:@collection)
+        else
+          false
+        end
+      end
+
+    end
+
+  end
+end

data/lib/threatinator/model/observables/fqdn_collection.rb

@@ -0,0 +1,15 @@
+require 'threatinator/model/collection'
+require 'domain_name_validator'
+
+module Threatinator
+  module Model
+    module Observables
+      class FqdnCollection < Threatinator::Model::Collection
+        def valid_member?(v)
+          #v.is_a?(::String)
+          ::DomainNameValidator.new.validate(v.to_s)
+        end
+      end
+    end
+  end
+end

data/lib/threatinator/model/observables/ipv4.rb

@@ -0,0 +1,33 @@
+require 'threatinator/model/base'
+require 'ip'
+require 'equalizer'
+
+module Threatinator
+  module Model
+    module Observables
+      class Ipv4 < Threatinator::Model::Base
+        include Equalizer.new(:ipv4)
+        attr_reader :ipv4
+
+        validates_each :ipv4 do |record, attr, value|
+          if value.is_a?(::IP::V4)
+            record.errors.add(attr, 'prefix length is not 32 bits') unless value.pfxlen == 32
+          else
+            record.errors.add(attr, 'not an IP::V4 object')
+          end
+        end
+
+        # @param [Hash] opts
+        # @option opts [IP::V4] :ipv4 An ipv4 object
+        def initialize(opts = {})
+          @ipv4 = opts.delete(:ipv4)
+          super()
+        end
+
+        def to_s
+          return ipv4.to_s
+        end
+      end
+    end
+  end
+end

data/lib/threatinator/model/observables/ipv4_collection.rb

@@ -0,0 +1,14 @@
+require 'threatinator/model/collection'
+require 'threatinator/model/observables/ipv4'
+
+module Threatinator
+  module Model
+    module Observables
+      class Ipv4Collection < Threatinator::Model::Collection
+        def valid_member?(v)
+          v.kind_of?(Threatinator::Model::Observables::Ipv4)
+        end
+      end
+    end
+  end
+end

data/lib/threatinator/model/observables/url_collection.rb

@@ -0,0 +1,16 @@
+require 'threatinator/model/collection'
+require 'addressable/uri'
+
+module Threatinator
+  module Model
+    module Observables
+      class UrlCollection < Threatinator::Model::Collection
+        def valid_member?(v)
+          v.is_a?(::Addressable::URI) &&
+            v.absolute?
+        end
+      end
+    end
+  end
+end
+

data/lib/threatinator/model/validations/type.rb

@@ -0,0 +1,21 @@
+require 'active_model'
+require 'active_model/validations'
+
+module Threatinator
+  module Model
+    module Validations
+      class TypeValidator < ActiveModel::EachValidator
+        def initialize(options)
+          @type = options.delete(:with)
+          super
+        end
+
+        def validate_each(record, name, value)
+          unless value.is_a?(@type)
+            record.errors.add name, "Expected to be #{@type}, got #{value.class} #{value.inspect}"
+          end
+        end
+      end
+    end
+  end
+end 

data/lib/threatinator/model/validations.rb

@@ -0,0 +1 @@
+require 'threatinator/model/validations/type'

data/lib/threatinator/output.rb

@@ -0,0 +1,50 @@
+require 'threatinator/config/base'
+
+module Threatinator
+  class Output
+    def initialize(config)
+    end
+
+    def handle_event(event)
+      #:nocov:
+      raise NotImplementedError.new("#{self.class}#handle_event is not implemented")
+      #:nocov:
+    end
+
+    def finish
+      #:nocov:
+      raise NotImplementedError.new("#{self.class}#finish is not implemented")
+      #:nocov:
+    end
+
+    class Config < Threatinator::Config::Base
+    end
+  end
+
+  class FileBasedOutput < Output
+    attr_reader :output_io
+    protected :output_io
+
+    def initialize(config)
+      super(config)
+      if io = config.io
+        @output_io = io
+      elsif filename = config.filename
+        @output_io = File.open(filename, 'w:UTF-8')
+      else
+        @output_io = $stdout.dup
+      end
+    end
+
+    def finish
+      @output_io.close
+    end
+
+    class Config < superclass::Config
+      attribute :filename, String, 
+        description: "Path to the file where output will be written"
+
+      attribute :io
+    end
+  end
+end

data/lib/threatinator/parser.rb

@@ -0,0 +1,23 @@
+module Threatinator
+  class Parser
+    # @param [Hash] opts An options hash. See subclasses for details.
+    def initialize(opts = {})
+    end
+
+    # Runs the parser against the provided io, yielding records.
+    # @param [IO] io The IO to be parsed.
+    def run(io)
+      raise NotImplementedError.new("#{self.class}#run not implemented!")
+    end
+
+    def ==(other)
+      true
+    end
+
+    def eql?(other)
+      self.class == other.class &&
+        self == other
+    end
+  end
+end
+

data/lib/threatinator/parsers/csv/parser.rb

@@ -0,0 +1,77 @@
+require 'threatinator/record'
+require 'threatinator/parser'
+require 'csv'
+
+module Threatinator
+  module Parsers
+    module CSV
+      # Parses an IO, yielding a record with a CSV::Row.
+      class Parser < Threatinator::Parser
+        attr_reader :csv_opts, :row_separator, :col_separator, :headers
+
+        # @param [Hash] opts 
+        # @option opts [String, :auto] :row_separator A string that represent the row
+        #  separator. Identical to ::CSV.new's :row_sep.
+        # @option opts [String] :col_separator A string that represents the column
+        #  separator. Identical to ::CSV.new's :col_sep.
+        # @option opts [Array<String>, :first_row, true, false] :headers The header
+        #  configuration. Identical to ::CSV.new's :headers. 
+        # @option opts [Hash] :csv_opts A hash of options that will be passed to
+        #  Ruby's CSV.new. 
+        # @see ::CSV
+        def initialize(opts = {})
+          @csv_opts = {}.merge(opts.delete(:csv_opts) || {})
+          @row_separator = opts.delete(:row_separator)
+          @col_separator = opts.delete(:col_separator)
+          @headers = opts.delete(:headers)
+
+          super(opts)
+        end
+
+        def ==(other)
+          @csv_opts == other.csv_opts &&
+            @row_separator == other.row_separator &&
+            @col_separator == other.col_separator &&
+            @headers == other.headers &&
+            super(other)
+        end
+
+        def _build_csv_opts
+          opts = {}.merge(@csv_opts)
+          opts[:return_headers] = true
+          opts[:row_sep] = @row_separator unless @row_separator.nil?
+          opts[:col_sep] = @col_separator unless @col_separator.nil?
+          opts[:headers] = @headers unless @headers.nil?
+          opts
+        end
+
+        # @param [IO] io
+        # @yield [record] Gives one line to the block
+        # @yieldparam record [Record] a record
+        def run(io)
+          lineno = 1
+          previous_pos = io.pos
+          csv = ::CSV.new(io, _build_csv_opts())
+          csv.each do |row|
+            begin
+              if row.kind_of?(::CSV::Row)
+                next if row.header_row?
+                row = row.to_hash
+              end
+
+              yield Record.new(row, 
+                               line_number: lineno, 
+                               pos_start: previous_pos, 
+                               pos_end: io.pos)
+
+            ensure
+              previous_pos = io.pos
+              lineno += 1
+            end
+          end
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/threatinator/parsers/csv.rb

@@ -0,0 +1,7 @@
+module Threatinator
+  module Parsers
+    module CSV
+      require 'threatinator/parsers/csv/parser'
+    end
+  end
+end

data/lib/threatinator/parsers/getline/parser.rb

@@ -0,0 +1,45 @@
+require 'threatinator/record'
+require 'threatinator/parser'
+
+module Threatinator
+  module Parsers
+    module Getline
+      # Parses an IO, yielding each 'line' of data as deliniated by :separator.
+      # The text matching :separator will be included.
+      class Parser < Threatinator::Parser
+        attr_reader :separator
+
+        # @param [Hash] opts 
+        # @option opts [String] :separator ("\n") A character that will be used
+        #  to detect the end of a line.
+        def initialize(opts = {})
+          @separator = opts.delete(:separator) || "\n"
+          unless @separator.length == 1
+            raise ArgumentError.new(":separator must be exactly one character long")
+          end
+          super(opts)
+        end
+
+        def ==(other)
+          @separator == other.separator &&
+            super(other)
+        end
+
+        # @param [IO] io The IO to be parsed
+        # @yield [line] Gives one line to the block
+        # @yieldparam line [String] a line from the IO stream.
+        def run(io)
+          return enum_for(:each) unless block_given?
+          lineno = 1
+          while str = io.gets(@separator)
+            return if str.nil?
+            pos_start = io.pos - str.length
+            yield Record.new(str, line_number: lineno, pos_start: pos_start, pos_end: io.pos)
+            lineno += 1
+          end
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/threatinator/parsers/getline.rb

@@ -0,0 +1,8 @@
+module Threatinator
+  module Parsers
+    module Getline
+      require 'threatinator/parsers/getline/parser'
+    end
+  end
+end
+

data/lib/threatinator/parsers/json/adapters/oj.rb

@@ -0,0 +1,65 @@
+require 'threatinator/parsers/json'
+require 'threatinator/exceptions'
+require 'oj'
+
+module Threatinator::Parsers::JSON::Adapters
+  class Oj < ::Oj::ScHandler
+    def initialize
+      @root = nil
+      @depth = 0
+    end
+
+    def run(io, &callback)
+      @callback = callback
+      begin
+        ::Oj.sc_parse(self, io)
+      rescue ::Oj::ParseError => e
+        raise Threatinator::Exceptions::ParseError.new(e)
+      end
+    end
+
+    def do_callback(data, key = nil)
+      @callback.call(data, key: key)
+    end
+
+    def hash_start
+      ret = {}
+      @depth += 1
+      @root = ret if @root.nil?
+      ret
+    end
+
+    def hash_set(h,k,v)
+      if @depth == 1
+        do_callback(v, k)
+      else 
+        h[k] = v
+      end
+      v
+    end
+
+    def hash_end
+      @depth -= 1
+    end
+
+    def array_start
+      ret = []
+      @depth += 1
+      @root = ret if @root.nil?
+      ret
+    end
+
+    def array_append(a,v)
+      if @depth == 1
+        do_callback(v)
+      else 
+        a << v
+      end
+    end
+
+    def array_end
+      @depth -= 1
+    end
+  end
+end
+

data/lib/threatinator/parsers/json/parser.rb

@@ -0,0 +1,45 @@
+require 'threatinator/parser'
+require 'threatinator/parsers/json/record'
+
+module Threatinator
+  module Parsers
+    module JSON
+      class Parser < Threatinator::Parser
+        def initialize(opts = {})
+          @adapter_class = self.class.adapter_class
+          super(opts)
+        end
+
+        # Detects the platform, loads the JSON adapter, and returns the 
+        # adapter's class.
+        def self.adapter_class
+          if defined?(RUBY_ENGINE) && RUBY_ENGINE == 'jruby'
+            #:nocov:
+            raise "JSON parsing not supported for JRuby"
+            #:nocov:
+          else 
+            require 'threatinator/parsers/json/adapters/oj'
+            return Threatinator::Parsers::JSON::Adapters::Oj
+          end
+        end
+
+        def ==(other)
+          super(other)
+        end
+
+        # @param [IO] io
+        # @yield [record] Gives one line to the block
+        # @yieldparam record [Threatinator::Parsers::JSON::Record] a record
+        def run(io)
+          adapter = @adapter_class.new
+          callback = lambda do |object, opts = {}|
+            yield Threatinator::Parsers::JSON::Record.new(object, opts)
+          end
+          adapter.run(io, &callback)
+          nil
+        end
+
+      end
+    end
+  end
+end

data/lib/threatinator/parsers/json/record.rb

@@ -0,0 +1,20 @@
+require 'threatinator/record'
+
+module Threatinator
+  module Parsers
+    module JSON
+      class Record < Threatinator::Record
+        attr_reader :key
+        def initialize(object, opts = {})
+          @key = opts.delete(:key)
+          super(object, opts)
+        end
+
+        def ==(other)
+          @key == other.key &&
+            super(other)
+        end
+      end
+    end
+  end
+end

data/lib/threatinator/parsers/json.rb

@@ -0,0 +1,8 @@
+module Threatinator
+  module Parsers
+    module JSON
+      require 'threatinator/parsers/json/parser'
+    end
+  end
+end
+

data/lib/threatinator/parsers/xml/node.rb

@@ -0,0 +1,79 @@
+module Threatinator
+  module Parsers
+    module XML
+      class Node
+        attr_accessor :text
+        attr_reader :name
+        attr_reader :attrs
+        attr_reader :children
+
+        # @param [String, Symbol] name
+        # @param [Hash] opts 
+        # @option opts [String] :text The text
+        # @option opts [Hash] :attrs The attributes
+        # @option opts [Array<Threatinator::Parsers::XML::Node>] :children An array 
+        #   of child child nodes that belong to this node.
+        def initialize(name, opts = {})
+          unless name.kind_of?(::Symbol) or name.kind_of?(::String)
+            raise TypeError.new("name must be a String or a Symbol")
+          end
+
+          @name = name.to_sym
+          @text = opts.delete(:text) || ""
+          unless @text.kind_of?(::String)
+            raise TypeError.new(":text must be a String")
+          end
+          @attrs = opts.delete(:attrs) || {}
+          unless @attrs.kind_of?(::Hash)
+            raise TypeError.new(":text must be a Hash")
+          end
+
+          @children = {}
+          if _children = opts.delete(:children)
+            _children.each do |child|
+              add_child(child)
+            end
+          end
+        end
+
+        def ==(other)
+          @name == other.name &&
+            @attrs == other.attrs &&
+            @text == other.text &&
+            @children == other.children
+        end
+
+        def eql?(other)
+          other.kind_of?(self.class) &&
+            self == other
+        end
+
+        # @return [Integer] the number of children
+        def num_children
+          @children.values.inject(0) {|total, child_set| total + child_set.count}
+        end
+
+        # @param [String, Symbol] name The name of the child element
+        # @return [Array<Node>] An array containing all the child nodes for the given
+        #  name. The array will be empty if there are no children by the given name.
+        def [](name)
+          @children[name.to_sym] || []
+        end
+
+        # @return [Array<Symbol>] an array containing all the names of child elements
+        def child_names
+          @children.keys
+        end
+
+        private
+        def add_child(child)
+          name = child.name
+          unless child_set = @children[name]
+            child_set = @children[name] = []
+          end
+          child_set << child
+        end
+      end
+    end
+  end
+end