mihari 5.6.1 → 5.7.0

data/lib/mihari/mixins/falsepositive.rb

@@ -2,6 +2,9 @@
 
 module Mihari
   module Mixins
+    #
+    # False positive mixins
+    #
     module FalsePositive
       include Memist::Memoizable
 

data/lib/mihari/mixins/refang.rb

@@ -2,6 +2,9 @@
 
 module Mihari
   module Mixins
+    #
+    # Refang mixin
+    #
     module Refang
       #
       # Refang defanged indicator

data/lib/mihari/mixins/retriable.rb

@@ -2,6 +2,9 @@
 
 module Mihari
   module Mixins
+    #
+    # Retriable mixin
+    #
     module Retriable
       DEFAULT_ON = [
         Errno::ECONNRESET,
@@ -11,8 +14,9 @@ module Mihari
         Timeout::Error,
         RetryableError,
         NetworkError,
-        TimeoutError
-      ]
+        TimeoutError,
+        StatusCodeError
+      ].freeze
 
       #
       # Retry on error

data/lib/mihari/models/alert.rb

@@ -1,88 +1,93 @@
 # frozen_string_literal: true
 
 module Mihari
-  class Alert < ActiveRecord::Base
-    has_many :taggings, dependent: :destroy
-    has_many :artifacts, dependent: :destroy
-    has_many :tags, through: :taggings
-
-    belongs_to :rule
-
-    class << self
-      #
-      # Search alerts
-      #
-      # @param [Structs::Filters::Alert::SearchFilterWithPagination] filter
-      #
-      # @return [Array<Alert>]
-      #
-      def search(filter)
-        limit = filter.limit.to_i
-        raise ArgumentError, "limit should be bigger than zero" unless limit.positive?
-
-        page = filter.page.to_i
-        raise ArgumentError, "page should be bigger than zero" unless page.positive?
-
-        offset = (page - 1) * limit
-
-        relation = build_relation(filter.without_pagination)
-
-        alert_ids = relation.limit(limit).offset(offset).order(id: :desc).pluck(:id).uniq
-        eager_load(:artifacts, :tags).where(id: [alert_ids]).order(id: :desc)
-      end
-
-      #
-      # Count alerts
-      #
-      # @param [String, nil] artifact_data
-      #
-      # @return [Integer]
-      #
-      def count(filter)
-        relation = build_relation(filter)
-        relation.distinct("alerts.id").count
-      end
+  module Models
+    #
+    # Alert model
+    #
+    class Alert < ActiveRecord::Base
+      has_many :taggings, dependent: :destroy
+      has_many :artifacts, dependent: :destroy
+      has_many :tags, through: :taggings
+
+      belongs_to :rule
+
+      class << self
+        #
+        # Search alerts
+        #
+        # @param [Mihari::Structs::Filters::Alert::SearchFilterWithPagination] filter
+        #
+        # @return [Array<Alert>]
+        #
+        def search(filter)
+          limit = filter.limit.to_i
+          raise ArgumentError, "limit should be bigger than zero" unless limit.positive?
+
+          page = filter.page.to_i
+          raise ArgumentError, "page should be bigger than zero" unless page.positive?
+
+          offset = (page - 1) * limit
+
+          relation = build_relation(filter.without_pagination)
+
+          alert_ids = relation.limit(limit).offset(offset).order(id: :desc).pluck(:id).uniq
+          eager_load(:artifacts, :tags).where(id: [alert_ids]).order(id: :desc)
+        end
 
-      private
-
-      #
-      # @param [Structs::Filters::Alert::SearchFilter] filter
-      #
-      # @return [Array<Integer>]
-      #
-      def get_artifact_ids_by_filter(filter)
-        artifact_ids = []
-
-        if filter.artifact_data
-          artifact = Artifact.where(data: filter.artifact_data)
-          artifact_ids = artifact.pluck(:id)
-          # set invalid ID if nothing is matched with the filters
-          artifact_ids = [-1] if artifact_ids.empty?
+        #
+        # Count alerts
+        #
+        # @param [Mihari::Structs::Filters::Alert::SearchFilter] filter
+        #
+        # @return [Integer]
+        #
+        def count(filter)
+          relation = build_relation(filter)
+          relation.distinct("alerts.id").count
         end
 
-        artifact_ids
-      end
+        private
 
-      #
-      # @param [Structs::Filters::Alert::SearchFilter] filter
-      #
-      # @return [Mihari::Alert]
-      #
-      def build_relation(filter)
-        artifact_ids = get_artifact_ids_by_filter(filter)
+        #
+        # @param [Mihari::Structs::Filters::Alert::SearchFilter] filter
+        #
+        # @return [Array<Integer>]
+        #
+        def get_artifact_ids_by_filter(filter)
+          artifact_ids = []
 
-        relation = self
-        relation = relation.includes(:artifacts, :tags)
+          if filter.artifact_data
+            artifact = Artifact.where(data: filter.artifact_data)
+            artifact_ids = artifact.pluck(:id)
+            # set invalid ID if nothing is matched with the filters
+            artifact_ids = [-1] if artifact_ids.empty?
+          end
 
-        relation = relation.where(artifacts: { id: artifact_ids }) unless artifact_ids.empty?
-        relation = relation.where(tags: { name: filter.tag_name }) if filter.tag_name
+          artifact_ids
+        end
+
+        #
+        # @param [Mihari::Structs::Filters::Alert::SearchFilter] filter
+        #
+        # @return [Mihari::Models::Alert]
+        #
+        def build_relation(filter)
+          artifact_ids = get_artifact_ids_by_filter(filter)
+
+          relation = self
+          relation = relation.includes(:artifacts, :tags)
 
-        relation = relation.where(rule_id: filter.rule_id) if filter.rule_id
+          relation = relation.where(artifacts: { id: artifact_ids }) unless artifact_ids.empty?
+          relation = relation.where(tags: { name: filter.tag_name }) if filter.tag_name
 
-        relation = relation.where("alerts.created_at >= ?", filter.from_at) if filter.from_at
-        relation = relation.where("alerts.created_at <= ?", filter.to_at) if filter.to_at
+          relation = relation.where(rule_id: filter.rule_id) if filter.rule_id
 
-        relation
+          relation = relation.where("alerts.created_at >= ?", filter.from_at) if filter.from_at
+          relation = relation.where("alerts.created_at <= ?", filter.to_at) if filter.to_at
+
+          relation
+        end
       end
     end
   end

data/lib/mihari/models/artifact.rb

@@ -1,229 +1,237 @@
 # frozen_string_literal: true
 
-class ArtifactValidator < ActiveModel::Validator
-  def validate(record)
-    return if record.data_type
-
-    record.errors.add :data, "#{record.data} is not supported"
-  end
-end
-
 module Mihari
-  class Artifact < ActiveRecord::Base
-    belongs_to :alert
-
-    has_one :autonomous_system, dependent: :destroy
-    has_one :geolocation, dependent: :destroy
-    has_one :whois_record, dependent: :destroy
-
-    has_many :cpes, dependent: :destroy
-    has_many :dns_records, dependent: :destroy
-    has_many :ports, dependent: :destroy
-    has_many :reverse_dns_names, dependent: :destroy
+  module Models
+    #
+    # Artifact validator
+    #
+    class ArtifactValidator < ActiveModel::Validator
+      def validate(record)
+        return if record.data_type
 
-    include ActiveModel::Validations
+        record.errors.add :data, "#{record.data} is not supported"
+      end
+    end
 
-    validates_with ArtifactValidator
+    #
+    # Artifact model
+    #
+    class Artifact < ActiveRecord::Base
+      belongs_to :alert
 
-    # @return [Array<Mihari::Tag>] Tags
-    attr_accessor :tags
+      has_one :autonomous_system, dependent: :destroy
+      has_one :geolocation, dependent: :destroy
+      has_one :whois_record, dependent: :destroy
 
-    # @return [String, nil] Rule ID
-    attr_accessor :rule_id
+      has_many :cpes, dependent: :destroy
+      has_many :dns_records, dependent: :destroy
+      has_many :ports, dependent: :destroy
+      has_many :reverse_dns_names, dependent: :destroy
 
-    def initialize(*args, **kwargs)
-      attrs = args.first || kwargs
-      data_ = attrs[:data]
+      include ActiveModel::Validations
 
-      raise TypeError if data_.is_a?(Array) || data_.is_a?(Hash)
+      validates_with ArtifactValidator
 
-      super(*args, **kwargs)
+      # @return [Array<Mihari::Tag>] Tags
+      attr_accessor :tags
 
-      self.data_type = TypeChecker.type(data)
+      # @return [String, nil] Rule ID
+      attr_accessor :rule_id
 
-      @tags = []
-      @rule_id = ""
-    end
+      def initialize(*args, **kwargs)
+        attrs = args.first || kwargs
+        data_ = attrs[:data]
 
-    #
-    # Check uniqueness of artifact
-    #
-    # @param [Time, nil] base_time Base time to check decaying
-    # @param [Integer, nil] artifact_lifetime Artifact lifetime (TTL) in seconds
-    #
-    # @return [Boolean] true if it is unique. Otherwise false.
-    #
-    def unique?(base_time: nil, artifact_lifetime: nil)
-      artifact = self.class.joins(:alert).where(
-        data: data,
-        alert: { rule_id: rule_id }
-      ).order(created_at: :desc).first
-      return true if artifact.nil?
+        raise TypeError if data_.is_a?(Array) || data_.is_a?(Hash)
 
-      # check whether the artifact is decayed or not
-      return false if artifact_lifetime.nil?
+        super(*args, **kwargs)
 
-      # use the current UTC time if base_time is not given (for testing)
-      base_time ||= Time.now.utc
+        self.data_type = TypeChecker.type(data)
 
-      decayed_at = base_time - (artifact_lifetime || -1).seconds
-      artifact.created_at < decayed_at
-    end
+        @tags = []
+        @rule_id = ""
+      end
 
-    #
-    # Enrich(add) whois record
-    #
-    # @param [Mihari::Enrichers::Whois] enricher
-    #
-    def enrich_whois(enricher = Enrichers::Whois.new)
-      return unless can_enrich_whois?
+      #
+      # Check uniqueness of artifact
+      #
+      # @param [Time, nil] base_time Base time to check decaying
+      # @param [Integer, nil] artifact_lifetime Artifact lifetime (TTL) in seconds
+      #
+      # @return [Boolean] true if it is unique. Otherwise false.
+      #
+      def unique?(base_time: nil, artifact_lifetime: nil)
+        artifact = self.class.joins(:alert).where(
+          data: data,
+          alert: { rule_id: rule_id }
+        ).order(created_at: :desc).first
+        return true if artifact.nil?
+
+        # check whether the artifact is decayed or not
+        return false if artifact_lifetime.nil?
+
+        # use the current UTC time if base_time is not given (for testing)
+        base_time ||= Time.now.utc
+
+        decayed_at = base_time - (artifact_lifetime || -1).seconds
+        artifact.created_at < decayed_at
+      end
 
-      self.whois_record = WhoisRecord.build_by_domain(normalize_as_domain(data), enricher: enricher)
-    end
+      #
+      # Enrich(add) whois record
+      #
+      # @param [Mihari::Enrichers::Whois] enricher
+      #
+      def enrich_whois(enricher = Enrichers::Whois.new)
+        return unless can_enrich_whois?
 
-    #
-    # Enrich(add) DNS records
-    #
-    # @param [Mihari::Enrichers::GooglePublicDNS] enricher
-    #
-    def enrich_dns(enricher = Enrichers::GooglePublicDNS.new)
-      return unless can_enrich_dns?
+        self.whois_record = WhoisRecord.build_by_domain(normalize_as_domain(data), enricher: enricher)
+      end
 
-      self.dns_records = DnsRecord.build_by_domain(normalize_as_domain(data), enricher: enricher)
-    end
+      #
+      # Enrich(add) DNS records
+      #
+      # @param [Mihari::Enrichers::GooglePublicDNS] enricher
+      #
+      def enrich_dns(enricher = Enrichers::GooglePublicDNS.new)
+        return unless can_enrich_dns?
 
-    #
-    # Enrich(add) reverse DNS names
-    #
-    # @param [Mihari::Enrichers::Shodan] enricher
-    #
-    def enrich_reverse_dns(enricher = Enrichers::Shodan.new)
-      return unless can_enrich_revese_dns?
+        self.dns_records = DnsRecord.build_by_domain(normalize_as_domain(data), enricher: enricher)
+      end
 
-      self.reverse_dns_names = ReverseDnsName.build_by_ip(data, enricher: enricher)
-    end
+      #
+      # Enrich(add) reverse DNS names
+      #
+      # @param [Mihari::Enrichers::Shodan] enricher
+      #
+      def enrich_reverse_dns(enricher = Enrichers::Shodan.new)
+        return unless can_enrich_revese_dns?
 
-    #
-    # Enrich(add) geolocation
-    #
-    # @param [Mihari::Enrichers::IPInfo] enricher
-    #
-    def enrich_geolocation(enricher = Enrichers::IPInfo.new)
-      return unless can_enrich_geolocation?
+        self.reverse_dns_names = ReverseDnsName.build_by_ip(data, enricher: enricher)
+      end
 
-      self.geolocation = Geolocation.build_by_ip(data, enricher: enricher)
-    end
+      #
+      # Enrich(add) geolocation
+      #
+      # @param [Mihari::Enrichers::IPInfo] enricher
+      #
+      def enrich_geolocation(enricher = Enrichers::IPInfo.new)
+        return unless can_enrich_geolocation?
 
-    #
-    # Enrich AS
-    #
-    # @param [Mihari::Enrichers::IPInfo] enricher
-    #
-    def enrich_autonomous_system(enricher = Enrichers::IPInfo.new)
-      return unless can_enrich_autonomous_system?
+        self.geolocation = Geolocation.build_by_ip(data, enricher: enricher)
+      end
 
-      self.autonomous_system = AutonomousSystem.build_by_ip(data, enricher: enricher)
-    end
+      #
+      # Enrich AS
+      #
+      # @param [Mihari::Enrichers::IPInfo] enricher
+      #
+      def enrich_autonomous_system(enricher = Enrichers::IPInfo.new)
+        return unless can_enrich_autonomous_system?
 
-    #
-    # Enrich ports
-    #
-    # @param [Mihari::Enrichers::Shodan] enricher
-    #
-    def enrich_ports(enricher = Enrichers::Shodan.new)
-      return unless can_enrich_ports?
+        self.autonomous_system = AutonomousSystem.build_by_ip(data, enricher: enricher)
+      end
 
-      self.ports = Port.build_by_ip(data, enricher: enricher)
-    end
+      #
+      # Enrich ports
+      #
+      # @param [Mihari::Enrichers::Shodan] enricher
+      #
+      def enrich_ports(enricher = Enrichers::Shodan.new)
+        return unless can_enrich_ports?
 
-    #
-    # Enrich CPEs
-    #
-    # @param [Mihari::Enrichers::Shodan] enricher
-    #
-    def enrich_cpes(enricher = Enrichers::Shodan.new)
-      return unless can_enrich_cpes?
+        self.ports = Port.build_by_ip(data, enricher: enricher)
+      end
 
-      self.cpes = CPE.build_by_ip(data, enricher: enricher)
-    end
+      #
+      # Enrich CPEs
+      #
+      # @param [Mihari::Enrichers::Shodan] enricher
+      #
+      def enrich_cpes(enricher = Enrichers::Shodan.new)
+        return unless can_enrich_cpes?
 
-    #
-    # Enrich all the enrichable relationships of the artifact
-    #
-    def enrich_all
-      enrich_autonomous_system
-      enrich_dns
-      enrich_geolocation
-      enrich_reverse_dns
-      enrich_whois
-      enrich_ports
-      enrich_cpes
-    end
+        self.cpes = CPE.build_by_ip(data, enricher: enricher)
+      end
 
-    ENRICH_METHODS_BY_ENRICHER = {
-      Enrichers::Whois => %i[
-        enrich_whois
-      ],
-      Enrichers::IPInfo => %i[
+      #
+      # Enrich all the enrichable relationships of the artifact
+      #
+      def enrich_all
         enrich_autonomous_system
+        enrich_dns
         enrich_geolocation
-      ],
-      Enrichers::Shodan => %i[
+        enrich_reverse_dns
+        enrich_whois
         enrich_ports
         enrich_cpes
-        enrich_reverse_dns
-      ],
-      Enrichers::GooglePublicDNS => %i[
-        enrich_dns
-      ]
-    }.freeze
+      end
 
-    #
-    # Enrich by name of enricher
-    #
-    # @param [Mihari::Enrichers::Base] enricher
-    #
-    def enrich_by_enricher(enricher)
-      methods = ENRICH_METHODS_BY_ENRICHER[enricher.class] || []
-      methods.each do |method|
-        send(method, enricher) if respond_to?(method)
+      ENRICH_METHODS_BY_ENRICHER = {
+        Enrichers::Whois => %i[
+          enrich_whois
+        ],
+        Enrichers::IPInfo => %i[
+          enrich_autonomous_system
+          enrich_geolocation
+        ],
+        Enrichers::Shodan => %i[
+          enrich_ports
+          enrich_cpes
+          enrich_reverse_dns
+        ],
+        Enrichers::GooglePublicDNS => %i[
+          enrich_dns
+        ]
+      }.freeze
+
+      #
+      # Enrich by name of enricher
+      #
+      # @param [Mihari::Enrichers::Base] enricher
+      #
+      def enrich_by_enricher(enricher)
+        methods = ENRICH_METHODS_BY_ENRICHER[enricher.class] || []
+        methods.each do |method|
+          send(method, enricher) if respond_to?(method)
+        end
       end
-    end
 
-    private
+      private
 
-    def normalize_as_domain(url_or_domain)
-      return url_or_domain if data_type == "domain"
+      def normalize_as_domain(url_or_domain)
+        return url_or_domain if data_type == "domain"
 
-      Addressable::URI.parse(url_or_domain).host
-    end
+        Addressable::URI.parse(url_or_domain).host
+      end
 
-    def can_enrich_whois?
-      %w[domain url].include?(data_type) && whois_record.nil?
-    end
+      def can_enrich_whois?
+        %w[domain url].include?(data_type) && whois_record.nil?
+      end
 
-    def can_enrich_dns?
-      %w[domain url].include?(data_type) && dns_records.empty?
-    end
+      def can_enrich_dns?
+        %w[domain url].include?(data_type) && dns_records.empty?
+      end
 
-    def can_enrich_revese_dns?
-      data_type == "ip" && reverse_dns_names.empty?
-    end
+      def can_enrich_revese_dns?
+        data_type == "ip" && reverse_dns_names.empty?
+      end
 
-    def can_enrich_geolocation?
-      data_type == "ip" && geolocation.nil?
-    end
+      def can_enrich_geolocation?
+        data_type == "ip" && geolocation.nil?
+      end
 
-    def can_enrich_autonomous_system?
-      data_type == "ip" && autonomous_system.nil?
-    end
+      def can_enrich_autonomous_system?
+        data_type == "ip" && autonomous_system.nil?
+      end
 
-    def can_enrich_ports?
-      data_type == "ip" && ports.empty?
-    end
+      def can_enrich_ports?
+        data_type == "ip" && ports.empty?
+      end
 
-    def can_enrich_cpes?
-      data_type == "ip" && cpes.empty?
+      def can_enrich_cpes?
+        data_type == "ip" && cpes.empty?
+      end
     end
   end
 end

data/lib/mihari/models/autonomous_system.rb

@@ -1,30 +1,35 @@
 # frozen_string_literal: true
 
 module Mihari
-  class AutonomousSystem < ActiveRecord::Base
-    belongs_to :artifact
+  module Models
+    #
+    # AS model
+    #
+    class AutonomousSystem < ActiveRecord::Base
+      belongs_to :artifact
 
-    class << self
-      include Dry::Monads[:result]
+      class << self
+        include Dry::Monads[:result]
 
-      #
-      # Build AS
-      #
-      # @param [String] ip
-      # @param [Mihari::Enrichers::IPInfo] enricher
-      #
-      # @return [Mihari::AutonomousSystem, nil]
-      #
-      def build_by_ip(ip, enricher: Enrichers::IPInfo.new)
-        result = enricher.query_result(ip).bind do |res|
-          value = res&.asn
-          if value.nil?
-            Success nil
-          else
-            Success new(asn: value)
+        #
+        # Build AS
+        #
+        # @param [String] ip
+        # @param [Mihari::Enrichers::IPInfo] enricher
+        #
+        # @return [Mihari::AutonomousSystem, nil]
+        #
+        def build_by_ip(ip, enricher: Enrichers::IPInfo.new)
+          result = enricher.result(ip).bind do |res|
+            value = res&.asn
+            if value.nil?
+              Success nil
+            else
+              Success new(asn: value)
+            end
           end
+          result.value_or nil
         end
-        result.value_or nil
       end
     end
   end