mihari 3.4.1 → 3.7.0

data/lib/mihari/models/alert.rb

@@ -18,8 +18,8 @@ module Mihari
       # @param [String, nil] source
       # @param [String, nil] tag_name
       # @param [String, nil] title
-      # @param [String, nil] from_at
-      # @param [String, nil] to_at
+      # @param [DateTime, nil] from_at
+      # @param [DateTime, nil] to_at
       # @param [Integer, nil] limit
       # @param [Integer, nil] page
       #
@@ -34,12 +34,22 @@ module Mihari
 
         offset = (page - 1) * limit
 
-        relation = build_relation(artifact_data: artifact_data, title: title, description: description, source: source, tag_name: tag_name, from_at: from_at, to_at: to_at)
+        relation = build_relation(
+          artifact_data: artifact_data,
+          title: title,
+          description: description,
+          source: source,
+          tag_name: tag_name,
+          from_at: from_at,
+          to_at: to_at
+        )
 
-        alerts = relation.limit(limit).offset(offset).order(id: :desc)
+        # TODO: improve queires
+        alert_ids = relation.limit(limit).offset(offset).order(id: :desc).pluck(:id).uniq
+        alerts = includes(:artifacts, :tags).where(id: [alert_ids]).order(id: :desc)
 
         alerts.map do |alert|
-          json = AlertSerializer.new(alert).as_json
+          json = Serializers::AlertSerializer.new(alert).as_json
           json[:artifacts] = json[:artifacts] || []
           json[:tags] = json[:tags] || []
           json
@@ -54,13 +64,21 @@ module Mihari
       # @param [String, nil] source
       # @param [String, nil] tag_name
       # @param [String, nil] title
-      # @param [String, nil] from_at
-      # @param [String, nil] to_at
+      # @param [DateTime, nil] from_at
+      # @param [DateTime, nil] to_at
       #
       # @return [Integer]
       #
       def count(artifact_data: nil, description: nil, source: nil, tag_name: nil, title: nil, from_at: nil, to_at: nil)
-        relation = build_relation(artifact_data: artifact_data, title: title, description: description, source: source, tag_name: tag_name, from_at: from_at, to_at: to_at)
+        relation = build_relation(
+          artifact_data: artifact_data,
+          title: title,
+          description: description,
+          source: source,
+          tag_name: tag_name,
+          from_at: from_at,
+          to_at: to_at
+        )
         relation.distinct("alerts.id").count
       end
 
@@ -68,8 +86,8 @@ module Mihari
 
       def build_relation(artifact_data: nil, title: nil, description: nil, source: nil, tag_name: nil, from_at: nil, to_at: nil)
         relation = self
-        relation = joins(:tags) if tag_name
-        relation = joins(:artifacts) if artifact_data
+
+        relation = relation.includes(:artifacts, :tags)
 
         relation = relation.where(artifacts: { data: artifact_data }) if artifact_data
         relation = relation.where(tags: { name: tag_name }) if tag_name

data/lib/mihari/models/artifact.rb

@@ -4,6 +4,7 @@ require "active_record"
 require "active_record/filter"
 require "active_support/core_ext/integer/time"
 require "active_support/core_ext/numeric/time"
+require "uri"
 
 class ArtifactValidator < ActiveModel::Validator
   def validate(record)
@@ -15,6 +16,15 @@ end
 
 module Mihari
   class Artifact < ActiveRecord::Base
+    belongs_to :alert
+
+    has_one :autonomous_system, dependent: :destroy
+    has_one :geolocation, dependent: :destroy
+    has_one :whois_record, dependent: :destroy
+
+    has_many :dns_records, dependent: :destroy
+    has_many :reverse_dns_names, dependent: :destroy
+
     include ActiveModel::Validations
 
     validates_with ArtifactValidator
@@ -44,5 +54,89 @@ module Mihari
       #                           within {ignore_threshold} days, do not ignore it
       artifact.created_at < days_before
     end
+
+    #
+    # Enrich(add) whois record
+    #
+    def enrich_whois
+      return unless can_enrich_whois?
+
+      self.whois_record = WhoisRecord.build_by_domain(normalize_as_domain(data))
+    end
+
+    #
+    # Enrich(add) DNS records
+    #
+    def enrich_dns
+      return unless can_enrich_dns?
+
+      self.dns_records = DnsRecord.build_by_domain(normalize_as_domain(data))
+    end
+
+    #
+    # Enrich(add) reverse DNS names
+    #
+    def enrich_reverse_dns
+      return unless can_enrich_revese_dns?
+
+      self.reverse_dns_names = ReverseDnsName.build_by_ip(data)
+    end
+
+    #
+    # Enrich(add) geolocation
+    #
+    def enrich_geolocation
+      return unless can_enrich_geolocation?
+
+      self.geolocation = Geolocation.build_by_ip(data)
+    end
+
+    #
+    # Enrich(add) geolocation
+    #
+    def enrich_autonomous_system
+      return unless can_enrich_autonomous_system?
+
+      self.autonomous_system = AutonomousSystem.build_by_ip(data)
+    end
+
+    #
+    # Enrich all the enrichable relationships of the artifact
+    #
+    def enrich_all
+      enrich_autonomous_system
+      enrich_dns
+      enrich_geolocation
+      enrich_reverse_dns
+      enrich_whois
+    end
+
+    private
+
+    def normalize_as_domain(url_or_domain)
+      return url_or_domain if data_type == "domain"
+
+      URI.parse(url_or_domain).host
+    end
+
+    def can_enrich_whois?
+      %w[domain url].include?(data_type) && whois_record.nil?
+    end
+
+    def can_enrich_dns?
+      %w[domain url].include?(data_type) && dns_records.empty?
+    end
+
+    def can_enrich_revese_dns?
+      data_type == "ip" && reverse_dns_names.empty?
+    end
+
+    def can_enrich_geolocation?
+      data_type == "ip" && geolocation.nil?
+    end
+
+    def can_enrich_autonomous_system?
+      data_type == "ip" && autonomous_system.nil?
+    end
   end
 end

data/lib/mihari/models/autonomous_system.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "active_record"
+
+module Mihari
+  class AutonomousSystem < ActiveRecord::Base
+    belongs_to :artifact
+
+    class << self
+      #
+      # Build AS
+      #
+      # @param [String] ip
+      #
+      # @return [Mihari::AutonomousSystem, nil]
+      #
+      def build_by_ip(ip)
+        res = Enrichers::IPInfo.query(ip)
+
+        unless res.nil?
+          return new(asn: res.asn)
+        end
+
+        nil
+      end
+    end
+  end
+end

data/lib/mihari/models/dns.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require "active_record"
+require "resolv"
+
+module Mihari
+  class DnsRecord < ActiveRecord::Base
+    belongs_to :artifact
+
+    class << self
+      #
+      # Build DNS records
+      #
+      # @param [String] domain
+      #
+      # @return [Array<Mihari::DnsRecord>]
+      #
+      def build_by_domain(domain)
+        resource_types = [
+          Resolv::DNS::Resource::IN::A,
+          Resolv::DNS::Resource::IN::AAAA,
+          Resolv::DNS::Resource::IN::CNAME,
+          Resolv::DNS::Resource::IN::TXT,
+          Resolv::DNS::Resource::IN::NS
+        ]
+
+        resource_types.map do |resource_type|
+          get_values domain, resource_type
+        rescue Resolv::ResolvError
+          nil
+        end.flatten.compact
+      end
+
+      private
+
+      def get_values(domain, resource_type)
+        resources = Resolv::DNS.new.getresources(domain, resource_type)
+        resource_name = resource_type.to_s.split("::").last
+
+        resources.map do |resource|
+          # A, AAAA
+          if resource.respond_to?(:address)
+            new(resource: resource_name, value: resource.address.to_s)
+          # CNAME, NS
+          elsif resource.respond_to?(:name)
+            new(resource: resource_name, value: resource.name.to_s)
+          # TXT
+          elsif resource.respond_to?(:data)
+            new(resource: resource_name, value: resource.data.to_s)
+          end
+        end.compact
+      end
+    end
+  end
+end

data/lib/mihari/models/geolocation.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require "active_record"
+require "normalize_country"
+
+module Mihari
+  class Geolocation < ActiveRecord::Base
+    belongs_to :artifact
+
+    class << self
+      #
+      # Build Geolocation
+      #
+      # @param [String] ip
+      #
+      # @return [Mihari::Geolocation, nil]
+      #
+      def build_by_ip(ip)
+        res = Enrichers::IPInfo.query(ip)
+
+        unless res.nil?
+          return new(country: NormalizeCountry(res.country_code, to: :short), country_code: res.country_code)
+        end
+
+        nil
+      end
+    end
+  end
+end

data/lib/mihari/models/reverse_dns.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require "active_record"
+require "resolv"
+
+module Mihari
+  class ReverseDnsName < ActiveRecord::Base
+    belongs_to :artifact
+
+    class << self
+      #
+      # Build reverse DNS names
+      #
+      # @param [String] ip
+      #
+      # @return [Array<Mihari::ReverseDnsName>]
+      #
+      def build_by_ip(ip)
+        names = Resolv.getnames(ip)
+        names.map { |name| new(name: name) }
+      rescue Resolv::ResolvError
+        []
+      end
+    end
+  end
+end

data/lib/mihari/models/whois.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+require "active_record"
+require "whois-parser"
+require "public_suffix"
+
+module Mihari
+  class WhoisRecord < ActiveRecord::Base
+    belongs_to :artifact
+
+    @memo = {}
+
+    class << self
+      #
+      # Build whois record
+      #
+      # @param [Stinrg] domain
+      #
+      # @return [WhoisRecord, nil]
+      #
+      def build_by_domain(domain)
+        domain = PublicSuffix.domain(domain)
+
+        # check memo
+        if @memo.key?(domain)
+          whois_record = @memo[domain]
+          # return clone of the record
+          return whois_record.dup
+        end
+
+        record = Whois.whois(domain)
+        parser = record.parser
+
+        return nil if parser.available?
+
+        whois_record = new(
+          domain: domain,
+          created_on: get_created_on(parser),
+          updated_on: get_updated_on(parser),
+          expires_on: get_expires_on(parser),
+          registrar: get_registrar(parser),
+          contacts: get_contacts(parser)
+        )
+        # set memo
+        @memo[domain] = whois_record
+        whois_record
+      rescue Whois::Error, Whois::ParserError, Timeout::Error
+        nil
+      end
+
+      private
+
+      #
+      # Get created_on
+      #
+      # @param [::Whois::Parser:] parser
+      #
+      # @return [Date, nil]
+      #
+      def get_created_on(parser)
+        parser.created_on
+      rescue ::Whois::AttributeNotImplemented
+        nil
+      end
+
+      #
+      # Get updated_on
+      #
+      # @param [::Whois::Parser:] parser
+      #
+      # @return [Date, nil]
+      #
+      def get_updated_on(parser)
+        parser.updated_on
+      rescue ::Whois::AttributeNotImplemented
+        nil
+      end
+
+      #
+      # Get expires_on
+      #
+      # @param [::Whois::Parser:] parser
+      #
+      # @return [Date, nil]
+      #
+      def get_expires_on(parser)
+        parser.expires_on
+      rescue ::Whois::AttributeNotImplemented
+        nil
+      end
+
+      #
+      # Get registrar
+      #
+      # @param [::Whois::Parser:] parser
+      #
+      # @return [Hash, nil]
+      #
+      def get_registrar(parser)
+        parser.registrar&.to_h
+      rescue ::Whois::AttributeNotImplemented
+        nil
+      end
+
+      #
+      # Get contacts
+      #
+      # @param [::Whois::Parser:] parser
+      #
+      # @return [Array[Hash], nil]
+      #
+      def get_contacts(parser)
+        parser.contacts.map(&:to_h)
+      rescue ::Whois::AttributeNotImplemented
+        nil
+      end
+    end
+  end
+end