fidius-evasiondb 0.0.1

data/lib/evasion-db/base.rb

@@ -0,0 +1,80 @@
+# Author::    FIDIUS (mailto:grp-fidius@tzi.de) 
+# License::   Distributes under the same terms as fidius-evasiondb Gem
+module FIDIUS
+  # EvasionDB is a knowledge database which provides information about idmef-events which are
+  # thrown by an ids like prelude. You can use this gem in two ways.
+  # One way is to generate knowledge via running exploits whithin metasploit. The other way is 
+  # require this gem and query existing knowledge estimate how loud an exploit might be.
+  #
+  module EvasionDB
+    # stores content of configuration yml
+    @@yml_config = nil
+    @@current_fetcher = nil
+    @@current_recorder = nil
+
+    # Configures EvasionDB. 
+    #
+    # @param [String] path to an yml-file containing db-connection settings for ids_db and evasion_db
+    # sample config can be created with 'fidius-evasiondb -e'
+    def self.config(yml_file)
+      if yml_file.class == String
+        raise "#{yml_file} does not exist" unless File.exists? File.expand_path(yml_file)
+        @@yml_config = YAML.load(File.read(yml_file))
+        evasion_db = @@yml_config['evasion_db']
+      elsif yml_file.class == Hash
+        # also react on connection settings given as hash
+        evasion_db = yml_file
+      else
+        raise "please input string or hash"
+      end
+      unless evasion_db
+        raise "no evasion_db part found in file"
+      else
+        #self.load_db_adapter(evasion_db['adapter'])
+        FIDIUS::EvasionDB::Knowledge::Connection.establish_connection evasion_db
+        FIDIUS::EvasionDB::Knowledge::Connection.connection
+      end
+    end
+
+    # Use a given recorder. Recorders are used to log packets while an exploit is running.
+    # Currently there is only the msf-recorder available. Use it by setting FIDIUS::EvasionDB.use_recorder "Msf-Recorder"
+    #
+    # @param [String] recordername
+    # @raise RuntimeError if recorder not found
+    def self.use_recoder(recorder_name)
+      raise "not configured. use FIDIUS::EvasionDB.config first" unless @@yml_config
+      @@current_recorder = Recorder.by_name(recorder_name)
+      raise "recorder #{recorder_name} not found" unless @@current_recorder
+    end
+
+    # Use a given fetcher. Fetchers are used to fetch idmef-events after an exploit is finished.
+    # Currently there is only the Prelude-Fetcher available. Use it by setting FIDIUS::EvasionDB.use_fetcher "PreludeDB"
+    #
+    # @param [String] fetcher_name
+    # @raise RuntimeError if fetcher not found
+    def self.use_fetcher(fetcher_name)
+      raise "not configured. use FIDIUS::EvasionDB.config first" unless @@yml_config
+      @@current_fetcher = Fetcher.by_name(fetcher_name)
+      @@current_fetcher.config(@@yml_config)
+      raise "fetcher #{recorder_name} not found" unless @@current_fetcher    
+    end
+
+    # Returns the current recorder
+    #
+    # @see #use_recorder
+    # @raise RuntimeError if recorder was not set
+    def self.current_recorder
+      raise "no recorder set. Use FIDIUS::EvasionDB.use_recorder" unless @@current_recorder
+      @@current_recorder
+    end
+
+    # Returns the current recorder
+    #
+    # @see #use_fetcher
+    # @raise RuntimeError if fetcher was not set
+    def self.current_fetcher
+      raise "no fetcher set. Use FIDIUS::EvasionDB.use_fetcher" unless @@current_fetcher
+      @@current_fetcher
+    end
+  end# module EvasionDB
+end# module FIDIUS

data/lib/evasion-db/idmef-fetchers/fetchers.rb

@@ -0,0 +1,67 @@
+module FIDIUS
+  module EvasionDB
+    def self.fetcher(name,&block)
+      FIDIUS::EvasionDB::Fetcher.new(name,&block)
+    end
+
+    def self.install_fetchers
+      $logger.debug "installing fetchers"
+      FIDIUS::EvasionDB::Fetcher.all.each do |fetcher|
+        fetcher.run_install
+      end
+    end
+
+    # A Fetcher is used to fetch events from an ids
+    class Fetcher
+      @@fetchers = []
+      attr_accessor :name
+      attr_accessor :local_ip
+
+      def initialize(name,&block)
+        self.instance_eval(&block)
+        @local_ip = nil
+        @name = name
+        @@fetchers << self
+      end
+
+      def install(&block)
+        $logger.debug "setting installblock"
+        @install = block
+      end
+
+      def run_install
+        raise "no install block given" unless @install
+        $logger.debug "run install of #{@name}"
+        @install.call
+      end
+
+      def config(conf)
+        raise "overwrite this"
+      end
+
+      def begin_record
+        raise "overwrite this"
+      end
+
+      def fetch_events
+        raise "overwrite this"
+      end
+
+      def self.all
+        @@fetchers
+      end
+
+      def self.by_name(name)
+        self.all.each do |fetcher|
+          return fetcher if fetcher.name == name
+        end
+        nil
+      end
+    end
+  end
+end
+
+Dir[File.join(File.dirname(__FILE__), "*/fetcher.rb")].each{|fetch_require|
+  $logger.debug "load #{fetch_require}"
+  require fetch_require
+} 

data/lib/evasion-db/idmef-fetchers/prelude-db/fetcher.rb

@@ -0,0 +1,8 @@
+$logger.debug "prelude-db"
+FIDIUS::EvasionDB.fetcher "PreludeDB" do
+  $logger.debug "prelude-db do"
+  install do
+    require (File.join File.dirname(__FILE__), 'lib', 'prelude_event_fetcher.rb')
+    self.extend FIDIUS::EvasionDB::PreludeEventFetcher
+  end
+end

data/lib/evasion-db/idmef-fetchers/prelude-db/lib/models/additional_data.rb

@@ -0,0 +1,16 @@
+module FIDIUS
+  module PreludeDB
+    # Wrapper for Prelude_AddionalData table
+    class AdditionalData < FIDIUS::PreludeDB::Connection
+      set_table_name "Prelude_AdditionalData"
+      def self.columns() @columns ||= []; end
+      def self.column(name, sql_type=nil, default=nil,null=true)
+        columns << ActiveRecord::ConnectionAdapters::Column.new(name.to_s,default,sql_type.to_s,null)
+      end
+
+      column :_message_ident, :bigint
+      column :meaning, :string
+      column :data, :longblob
+    end
+  end
+end

data/lib/evasion-db/idmef-fetchers/prelude-db/lib/models/address.rb

@@ -0,0 +1,8 @@
+module FIDIUS
+  module PreludeDB
+    # Wrapper for Prelude_Address table
+    class Address < FIDIUS::PreludeDB::Connection
+      set_table_name "Prelude_Address"
+    end
+  end
+end

data/lib/evasion-db/idmef-fetchers/prelude-db/lib/models/alert.rb

@@ -0,0 +1,46 @@
+module FIDIUS
+  module PreludeDB
+
+    # Wrapper for Prelude_Alert table
+    class Alert < FIDIUS::PreludeDB::Connection
+      has_one :detect_time, :class_name => 'DetectTime', :foreign_key => :_message_ident, :primary_key => :_ident
+      has_one :source_address, :class_name => 'Address', :foreign_key => :_message_ident, :primary_key => :_ident, :conditions => [ "Prelude_Address._parent_type = 'S'" ]
+      has_one :dest_address, :class_name => 'Address', :foreign_key => :_message_ident, :primary_key => :_ident, :conditions => [ "Prelude_Address._parent_type = 'T'" ]
+      has_one :source_port, :class_name => 'Service', :foreign_key => :_message_ident, :primary_key => :_ident, :conditions => [ "Prelude_Service._parent_type = 'S'" ]
+      has_one :dest_port, :class_name => 'Service', :foreign_key => :_message_ident, :primary_key => :_ident, :conditions => [ "Prelude_Service._parent_type = 'T'" ]
+      has_one :classification, :class_name => 'Classification', :foreign_key => :_message_ident, :primary_key => :_ident
+      has_one :analyzer, :class_name => 'Analyzer', :foreign_key => :_message_ident, :primary_key => :_ident
+      has_one :impact, :class_name => 'Impact', :foreign_key => :_message_ident, :primary_key => :_ident
+      has_one :payload, :class_name => 'AdditionalData', :foreign_key => :_message_ident, :primary_key => :_ident, :conditions=>["Prelude_AdditionalData.meaning='payload'"]
+      set_primary_key :_ident
+
+      def self.table_name
+        "Prelude_Alert"
+      end
+
+      def self.total_entries
+        sql = connection();
+	      sql.begin_db_transaction
+	      value = sql.execute("SELECT count(*) FROM Prelude_Alert;").fetch_row;
+	      sql.commit_db_transaction
+	      value[0].to_i;
+      end
+
+      def source_ip
+        source_address.address
+      end
+
+      def dest_ip
+         dest_address.address
+      end
+
+      def severity
+        return impact.severity
+      end
+
+      def payload_data
+        payload.data if payload != nil
+      end
+    end
+  end
+end

data/lib/evasion-db/idmef-fetchers/prelude-db/lib/models/analyzer.rb

@@ -0,0 +1,18 @@
+module FIDIUS
+  module PreludeDB
+    # Wrapper for Prelude_Analyzer table
+    class Analyzer < FIDIUS::PreludeDB::Connection
+      def self.columns() @columns ||= []; end
+      def self.column(name, sql_type=nil, default=nil,null=true)
+        columns << ActiveRecord::ConnectionAdapters::Column.new(name.to_s,default,sql_type.to_s,null)
+      end
+      column :model, :string
+      column :name, :string
+      column :_message_ident, :bigint
+
+      def self.table_name
+        "Prelude_Analyzer"
+      end
+    end
+  end
+end

data/lib/evasion-db/idmef-fetchers/prelude-db/lib/models/classification.rb

@@ -0,0 +1,10 @@
+module FIDIUS
+  module PreludeDB
+    # Wrapper for Prelude_Classification table
+    class Classification < FIDIUS::PreludeDB::Connection
+      def self.table_name
+        "Prelude_Classification"
+      end
+    end
+  end
+end

data/lib/evasion-db/idmef-fetchers/prelude-db/lib/models/connection.rb

@@ -0,0 +1,10 @@
+module FIDIUS
+  # This module provides active-record wrappers for an existing preludemanager database.
+  module PreludeDB
+    # Base class for all other PreludeManager Models.
+    # It is handy for establishing connection for all models 
+    class Connection < ActiveRecord::Base
+
+    end
+  end
+end

data/lib/evasion-db/idmef-fetchers/prelude-db/lib/models/detect_time.rb

@@ -0,0 +1,10 @@
+module FIDIUS
+  module PreludeDB
+    # Wrapper for Prelude_DetectTime table
+    class DetectTime < FIDIUS::PreludeDB::Connection
+      def self.table_name
+        "Prelude_DetectTime"
+      end
+    end
+  end
+end

data/lib/evasion-db/idmef-fetchers/prelude-db/lib/models/impact.rb

@@ -0,0 +1,19 @@
+module FIDIUS
+  module PreludeDB
+    # Wrapper for Prelude_Impact table
+    class Impact < FIDIUS::PreludeDB::Connection
+      def self.columns() @columns ||= []; end
+      def self.column(name, sql_type=nil, default=nil,null=true)
+        columns << ActiveRecord::ConnectionAdapters::Column.new(name.to_s,default,sql_type.to_s,null)
+      end
+
+      column :description, :string
+      column :severity, :string
+      column :_message_ident, :bigint
+
+      def self.table_name
+        "Prelude_Impact"
+      end
+    end
+  end
+end

data/lib/evasion-db/idmef-fetchers/prelude-db/lib/models/prelude_event.rb

@@ -0,0 +1,120 @@
+module FIDIUS
+  module PreludeDB
+    # Represents an IDMEF-Event which is distributed over multiple tables in PreludeManager
+    class PreludeEvent < FIDIUS::PreludeDB::Connection
+      has_many :annotated_events
+      def self.columns() @columns ||= []; end
+      def self.column(name, sql_type=nil, default=nil,null=true)
+        columns << ActiveRecord::ConnectionAdapters::Column.new(name.to_s,default,sql_type.to_s,null)
+      end
+
+      @prelude_alert = nil
+
+      def initialize(prelude_alert)
+        @prelude_alert = prelude_alert
+      end
+
+      def self.find_by_sql(query)
+
+      end
+      
+      def self.total_entries(options = nil)
+        return Alert.total_entries
+      end
+
+      def self.find(*args)
+        if args[0].is_a? Numeric
+          a = Alert.find(:all, :conditions => [ "_ident = ?", args[0] ])
+          return PreludeEvent.new a.first
+        else
+          case args[0]  
+            when :all
+              if args[1]
+                if(args[1][:conditions] == nil)
+                  args[1] = args[1].merge({:joins => [:detect_time,]})
+                  args[1] = args[1].merge({:order => 'time DESC'})
+                end
+              end
+              a = Alert.find(*args)
+              result = Array.new
+              a.each do |pa|
+                result.push PreludeEvent.new pa
+              end
+              return result
+            when :first
+              a = Alert.first
+              return PreludeEvent.new a
+            when :last
+              a = Alert.last
+              return PreludeEvent.new a
+            else  
+
+          end
+        end
+      end
+
+      def source_ip
+        return @prelude_alert.source_ip unless @prelude_alert.nil?
+        return "No Ref"
+      end
+
+      def dest_ip
+        return @prelude_alert.dest_ip unless @prelude_alert.nil?
+        return "No Ref"
+      end
+
+      def source_port
+        return @prelude_alert.source_port.port unless @prelude_alert.nil?
+        return "No Ref"
+      end
+
+      def dest_port
+        return @prelude_alert.dest_port.port unless @prelude_alert.nil?
+        return "No Ref"
+      end
+
+      def payload
+        return @prelude_alert.payload_data unless @prelude_alert.nil?
+      end
+
+      def detect_time
+       return @prelude_alert.detect_time.time unless @prelude_alert.nil?
+       return "No Ref"
+      end
+      def text
+        return @prelude_alert.classification.text  unless @prelude_alert.nil?
+        return "No Ref"
+      end
+      def severity
+        return @prelude_alert.severity  unless @prelude_alert.nil?
+        return "No Ref"
+      end
+      def analyzer_model
+        return @prelude_alert.analyzer.name.to_s  unless @prelude_alert.nil?
+        return "No Ref"
+      end
+
+      def id
+        return @prelude_alert._ident  unless @prelude_alert.nil?
+        return "No Ref"
+      end
+
+      def inspect
+        begin
+        return "PreludeEvent id: "+id.to_s+", source_ip: "+source_ip+" dest_ip: "+dest_ip+" severity: "+severity+" text: "+text+" analyzer_model: "+analyzer_model+" detect_time: "+detect_time.to_s+""
+        rescue
+          puts $!.message+":"+$!.backtrace.to_s
+        end
+      end
+
+      def to_s
+        "#{text}: #{source_ip}:#{source_port} -> #{dest_ip}:#{dest_port}"
+      end
+
+      def messageid
+        return @prelude_alert.messageid unless @prelude_alert.nil?
+        return "No Ref"
+      end
+    end
+  end
+end

data/lib/evasion-db/idmef-fetchers/prelude-db/lib/models/service.rb

@@ -0,0 +1,10 @@
+module FIDIUS
+  module PreludeDB
+    # Wrapper for Prelude_Service table
+    class Service < FIDIUS::PreludeDB::Connection
+      def self.table_name
+        "Prelude_Service"
+      end
+    end
+  end
+end

data/lib/evasion-db/idmef-fetchers/prelude-db/lib/patches/postgres_patch.rb

@@ -0,0 +1,12 @@
+module ActiveRecord
+  module ConnectionAdapters
+    # This class is a patch to make postgres play nice
+    class PostgreSQLAdapter < AbstractAdapter
+      # will fix quotation bug
+      def quote_table_name(name)
+        return name
+      end
+    end
+  end
+end
+

data/lib/evasion-db/idmef-fetchers/prelude-db/lib/prelude_event_fetcher.rb

@@ -0,0 +1,67 @@
+module FIDIUS
+  module EvasionDB
+    module PreludeEventFetcher
+      def config(conf)
+        $logger.debug "INIT PRELUDE EVENT FETCHER"
+ 	      ids_db = conf['ids_db']
+        raise "no ids_db part found" unless ids_db
+        FIDIUS::PreludeDB::Connection.establish_connection ids_db
+        connection = FIDIUS::PreludeDB::Connection.connection
+        $logger.debug "connection is: #{connection}"
+        require (File.join File.dirname(__FILE__), 'patches', 'postgres_patch.rb')
+      end
+
+      def begin_record
+        a = FIDIUS::PreludeDB::Alert.find(:first,:joins => [:detect_time],:order=>"time DESC")
+        last_event = FIDIUS::PreludeDB::PreludeEvent.new(a)
+        @start_time = last_event.detect_time
+      end
+
+      def get_events
+        raise "please begin_record before fetching" if @start_time == nil
+        res = Array.new
+        sleep 3
+        $logger.debug "alert.find(:all,:joins=>[:detect_time],time > #{@start_time})"
+        events = FIDIUS::PreludeDB::Alert.find(:all,:joins => [:detect_time],:order=>"time DESC",:conditions=>["time > :d",{:d => @start_time}])
+        $logger.debug "found #{events.size} events"
+        events.each do |event|
+          ev = FIDIUS::PreludeDB::PreludeEvent.new(event)
+          $logger.debug "Event #{ev.source_ip} -> #{ev.dest_ip}  local_ip:#{@local_ip}"
+          if @local_ip
+            if (ev.source_ip == @local_ip || ev.dest_ip == @local_ip)
+              $logger.debug "adding #{ev.inspect} to events "
+              res << ev  
+            end
+          else
+            $logger.debug "adding #{ev.inspect} to events "
+            res << ev
+          end
+        end
+        return res
+      end
+
+      def fetch_events(module_instance=nil)
+        result = []
+        events = get_events
+        events.each do |event|
+          idmef_event = FIDIUS::EvasionDB::Knowledge::IdmefEvent.create(:payload=>event.payload,:detect_time=>event.detect_time,
+                            :dest_ip=>event.dest_ip,:src_ip=>event.source_ip,
+                            :dest_port=>event.dest_port,:src_port=>event.source_port,
+                            :text=>event.text,:severity=>event.severity,
+                            :analyzer_model=>event.analyzer_model,:ident=>event.id)
+          result << idmef_event
+        end
+        return result
+      end
+
+    end
+  end
+end
+
+require (File.join File.dirname(__FILE__), 'models', 'connection.rb')
+
+Dir.glob(File.join File.dirname(__FILE__), 'models', '*.rb') do |rb|
+  $logger.debug "loading #{rb}"
+  require rb
+end
+

data/lib/evasion-db/idmef-fetchers/test-fetcher/fetcher.rb

@@ -0,0 +1,6 @@
+FIDIUS::EvasionDB.fetcher "TestFetcher" do
+  install do
+    require (File.join File.dirname(__FILE__), 'lib', 'test_fetcher.rb')
+    self.extend FIDIUS::EvasionDB::TestFetcher
+  end
+end

data/lib/evasion-db/idmef-fetchers/test-fetcher/lib/test_fetcher.rb

@@ -0,0 +1,19 @@
+module FIDIUS
+  module EvasionDB
+    # This module is only a sample how a custom fetcher could be implemented.
+    module TestFetcher
+      def config(conf)
+        $logger.debug "INIT Test FETCHER"
+      end
+
+      def begin_record
+        $logger.debug "test-fetcher begin record"
+      end
+
+      def fetch_events(*args)
+        $logger.debug "test-fetcher get events"
+        return []
+      end
+    end
+  end
+end

data/lib/evasion-db/knowledge/attack_module.rb

@@ -0,0 +1,41 @@
+require 'digest/md5'
+module FIDIUS::EvasionDB::Knowledge
+  # Represents an Attack
+  # in metasploit this would be exploits or auxiliaries
+  class AttackModule < FIDIUS::EvasionDB::Knowledge::Connection
+    has_many :idmef_events, :dependent=>:destroy
+    has_many :packets, :dependent=>:destroy
+    has_many :attack_options, :dependent=>:destroy
+    has_one :attack_payload, :dependent=>:destroy
+
+    def self.table_name
+      "attack_modules"
+    end
+
+    def self.find_or_create_by_name_and_options(name,options)
+      hash = Digest::MD5.hexdigest(options.to_s)
+      attack_module = self.find_or_create_by_name_and_options_hash(name,hash)
+      if attack_module.attack_options.size == 0
+        attack_module.options_hash = hash
+        $logger.debug "PAYLOAD: #{options['PAYLOAD']}"
+        attack_module.attack_payload = AttackPayload.new(:name=>options["Payload"]) if options["Payload"]
+        options.each do |k,v|
+          attack_module.attack_options << AttackOption.new(:option_key=>k,:option_value=>v)
+        end
+        attack_module.save
+      end
+      attack_module
+    end
+
+    def has_option(k,v)
+      attack_options.find_by_option_key_and_option_value(k,v) != nil
+    end
+
+    def has_options(options = {})
+      options.each do |k,v|
+        return false if !self.has_option(k,v)
+      end
+      true
+    end
+  end
+end

data/lib/evasion-db/knowledge/attack_option.rb

@@ -0,0 +1,12 @@
+module FIDIUS::EvasionDB::Knowledge
+  # every AttackModule can have multiple options
+  # these are key value pairs which are represented by this class
+  class AttackOption < FIDIUS::EvasionDB::Knowledge::Connection
+    belongs_to :attack_module
+
+    def self.table_name
+      "attack_options"
+    end
+
+  end
+end

data/lib/evasion-db/knowledge/attack_payload.rb

@@ -0,0 +1,13 @@
+module FIDIUS::EvasionDB::Knowledge
+  # everey attack can have a payload.
+  # in metasploit this could be meterpreter
+  class AttackPayload < FIDIUS::EvasionDB::Knowledge::Connection
+    belongs_to :attack_module
+    has_many :packets
+    has_many :idmef_events
+
+    def self.table_name
+      "attack_payloads"
+    end
+  end
+end

data/lib/evasion-db/knowledge/connection.rb

@@ -0,0 +1,7 @@
+module FIDIUS::EvasionDB::Knowledge
+  # base class for knowledge db.
+  # it is handy to establish connection for all models at once
+  class Connection < ActiveRecord::Base
+
+  end
+end 

data/lib/evasion-db/knowledge/idmef_event.rb

@@ -0,0 +1,21 @@
+module FIDIUS::EvasionDB::Knowledge
+  # model for local store generated idmef events and relate them to an executed attack
+  class IdmefEvent < FIDIUS::EvasionDB::Knowledge::Connection
+    belongs_to :attack_module
+    belongs_to :attack_payload
+
+    def self.table_name
+      "idmef_events"
+    end
+
+    def payload
+      return [] if self[:payload] == nil
+      self[:payload]
+    end
+
+    def payload_size
+      return self[:payload].size if self[:payload]
+      return 0
+    end
+  end
+end

data/lib/evasion-db/knowledge/packet.rb

@@ -0,0 +1,17 @@
+module FIDIUS::EvasionDB::Knowledge
+  # every attack module sends multiple packets which are stored within this model
+  class Packet < FIDIUS::EvasionDB::Knowledge::Connection
+    belongs_to :attack_module
+    belongs_to :attack_payload
+
+    def self.table_name
+      "packets"
+    end
+
+    def payload
+      return [] if self[:payload] == nil
+      self[:payload]
+    end
+
+  end
+end