mihari 6.3.0 → 7.0.1

data/lib/mihari/models/alert.rb

@@ -6,63 +6,31 @@ module Mihari
     # Alert model
     #
     class Alert < ActiveRecord::Base
-      has_many :taggings, dependent: :destroy
-      has_many :artifacts, dependent: :destroy
-      has_many :tags, through: :taggings
-
       belongs_to :rule
 
-      class << self
-        #
-        # Search alerts
-        #
-        # @param [Mihari::Structs::Filters::Alert::SearchFilterWithPagination] filter
-        #
-        # @return [Array<Alert>]
-        #
-        def search(filter)
-          limit = filter.limit.to_i
-          raise ArgumentError, "limit should be bigger than zero" unless limit.positive?
-
-          page = filter.page.to_i
-          raise ArgumentError, "page should be bigger than zero" unless page.positive?
-
-          offset = (page - 1) * limit
-
-          relation = build_relation(filter.without_pagination)
-          relation.limit(limit).offset(offset).order(id: :desc)
-        end
-
-        #
-        # Count alerts
-        #
-        # @param [Mihari::Structs::Filters::Alert::SearchFilter] filter
-        #
-        # @return [Integer]
-        #
-        def count(filter)
-          relation = build_relation(filter)
-          relation.distinct("alerts.id").count
-        end
-
-        private
-
-        #
-        # @param [Mihari::Structs::Filters::Alert::SearchFilter] filter
-        #
-        # @return [Mihari::Models::Alert]
-        #
-        def build_relation(filter)
-          relation = eager_load(:artifacts, :tags)
+      has_many :artifacts, dependent: :destroy
+      has_many :tags, through: :rule
+
+      include SearchCop
+      include Concerns::Searchable
+
+      search_scope :search do
+        attributes :id, :created_at, "rule.id", "rule.title", "rule.description"
+        attributes "artifact.data" => "artifacts.data"
+        attributes "artifact.data_type" => "artifacts.data_type"
+        attributes "artifact.source" => "artifacts.source"
+        attributes "artifact.query" => "artifacts.query"
+        attributes tag: "tags.name"
+      end
 
-          relation = relation.where(artifacts: { data: filter.artifact }) if filter.artifact
-          relation = relation.where(tags: { name: filter.tag }) if filter.tag
-          relation = relation.where(rule_id: filter.rule_id) if filter.rule_id
-          relation = relation.where("alerts.created_at >= ?", filter.from_at) if filter.from_at
-          relation = relation.where("alerts.created_at <= ?", filter.to_at) if filter.to_at
+      class << self
+        # @!method search_by_filter(filter)
+        #   @param [Mihari::Structs::Filters::Search] filter
+        #   @return [Array<Mihari::Models::Alert>]
 
-          relation
-        end
+        # @!method count_by_filter(filter)
+        #   @param [Mihari::Structs::Filters::Search] filter
+        #   @return [Integer]
       end
     end
   end

data/lib/mihari/models/artifact.rb

@@ -22,45 +22,47 @@ module Mihari
       has_one :autonomous_system, dependent: :destroy
       has_one :geolocation, dependent: :destroy
       has_one :whois_record, dependent: :destroy
+      has_one :rule, through: :alert
 
       has_many :cpes, dependent: :destroy
       has_many :dns_records, dependent: :destroy
       has_many :ports, dependent: :destroy
       has_many :reverse_dns_names, dependent: :destroy
+      has_many :tags, through: :alert
 
       include ActiveModel::Validations
+      include SearchCop
+      include Concerns::Searchable
+
+      search_scope :search do
+        attributes :id, :data, :data_type, :source, :query, :created_at, "alert.id", "rule.id", "rule.title",
+          "rule.description"
+        attributes tag: "tags.name"
+        attributes asn: "autonomous_system.asn"
+        attributes country_code: "geolocation.country_code"
+        attributes "dns_record.value": "dns_records.value"
+        attributes "dns_record.resource": "dns_records.resource"
+        attributes reverse_dns_name: "reverse_dns_names.name"
+        attributes cpe: "cpes.name"
+        attributes port: "ports.port"
+      end
 
       validates_with ArtifactValidator
 
-      # @return [Array<Mihari::Tag>] Tags
-      attr_accessor :tags
+      after_initialize :set_data_type, :set_rule_id, if: :new_record?
 
-      # @return [String, nil] Rule ID
+      # @return [String, nil]
       attr_accessor :rule_id
 
-      def initialize(*args, **kwargs)
-        attrs = args.first || kwargs
-        data_ = attrs[:data]
-
-        raise TypeError if data_.is_a?(Array) || data_.is_a?(Hash)
-
-        super(*args, **kwargs)
-
-        self.data_type = DataType.type(data)
-
-        @tags = []
-        @rule_id = ""
-      end
-
       #
-      # Check uniqueness of artifact
+      # Check uniqueness
       #
       # @param [Time, nil] base_time Base time to check decaying
-      # @param [Integer, nil] artifact_lifetime Artifact lifetime (TTL) in seconds
+      # @param [Integer, nil] artifact_ttl Artifact TTL in seconds
       #
       # @return [Boolean] true if it is unique. Otherwise false.
       #
-      def unique?(base_time: nil, artifact_lifetime: nil)
+      def unique?(base_time: nil, artifact_ttl: nil)
         artifact = self.class.joins(:alert).where(
           data: data,
           alert: { rule_id: rule_id }
@@ -68,27 +70,15 @@ module Mihari
         return true if artifact.nil?
 
         # check whether the artifact is decayed or not
-        return false if artifact_lifetime.nil?
+        return false if artifact_ttl.nil?
 
         # use the current UTC time if base_time is not given (for testing)
         base_time ||= Time.now.utc
 
-        decayed_at = base_time - (artifact_lifetime || -1).seconds
+        decayed_at = base_time - (artifact_ttl || -1).seconds
         artifact.created_at < decayed_at
       end
 
-      #
-      # Count artifacts
-      #
-      # @param [Mihari::Structs::Filters::Artifact::SearchFilter] filter
-      #
-      # @return [Integer]
-      #
-      def count(filter)
-        relation = build_relation(filter)
-        relation.distinct("artifact.id").count
-      end
-
       #
       # Enrich whois record
       #
@@ -97,7 +87,7 @@ module Mihari
       def enrich_whois(enricher = Enrichers::Whois.new)
         return unless can_enrich_whois?
 
-        self.whois_record = WhoisRecord.build_by_domain(normalize_as_domain(data), enricher: enricher)
+        self.whois_record = Services::WhoisRecordBuilder.call(domain, enricher: enricher)
       end
 
       #
@@ -108,7 +98,7 @@ module Mihari
       def enrich_dns(enricher = Enrichers::GooglePublicDNS.new)
         return unless can_enrich_dns?
 
-        self.dns_records = DnsRecord.build_by_domain(normalize_as_domain(data), enricher: enricher)
+        self.dns_records = Services::DnsRecordBuilder.call(domain, enricher: enricher)
       end
 
       #
@@ -119,7 +109,7 @@ module Mihari
       def enrich_reverse_dns(enricher = Enrichers::Shodan.new)
         return unless can_enrich_reverse_dns?
 
-        self.reverse_dns_names = ReverseDnsName.build_by_ip(data, enricher: enricher)
+        self.reverse_dns_names = Services::ReverseDnsNameBuilder.call(data, enricher: enricher)
       end
 
       #
@@ -127,10 +117,10 @@ module Mihari
       #
       # @param [Mihari::Enrichers::IPInfo] enricher
       #
-      def enrich_geolocation(enricher = Enrichers::IPInfo.new)
+      def enrich_geolocation(enricher = Enrichers::MMDB.new)
         return unless can_enrich_geolocation?
 
-        self.geolocation = Geolocation.build_by_ip(data, enricher: enricher)
+        self.geolocation = Services::GeolocationBuilder.call(data, enricher: enricher)
       end
 
       #
@@ -138,10 +128,10 @@ module Mihari
       #
       # @param [Mihari::Enrichers::IPInfo] enricher
       #
-      def enrich_autonomous_system(enricher = Enrichers::IPInfo.new)
+      def enrich_autonomous_system(enricher = Enrichers::MMDB.new)
         return unless can_enrich_autonomous_system?
 
-        self.autonomous_system = AutonomousSystem.build_by_ip(data, enricher: enricher)
+        self.autonomous_system = Services::AutonomousSystemBuilder.call(data, enricher: enricher)
       end
 
       #
@@ -152,7 +142,7 @@ module Mihari
       def enrich_ports(enricher = Enrichers::Shodan.new)
         return unless can_enrich_ports?
 
-        self.ports = Port.build_by_ip(data, enricher: enricher)
+        self.ports = Services::PortBuilder.call(data, enricher: enricher)
       end
 
       #
@@ -163,16 +153,16 @@ module Mihari
       def enrich_cpes(enricher = Enrichers::Shodan.new)
         return unless can_enrich_cpes?
 
-        self.cpes = CPE.build_by_ip(data, enricher: enricher)
+        self.cpes = Services::CPEBuilder.call(data, enricher: enricher)
       end
 
       #
       # Enrich all the enrichable relationships of the artifact
       #
       def enrich_all
-        enrich_autonomous_system ipinfo
+        enrich_autonomous_system mmdb
         enrich_dns
-        enrich_geolocation ipinfo
+        enrich_geolocation mmdb
         enrich_reverse_dns shodan
         enrich_whois
         enrich_ports shodan
@@ -183,7 +173,7 @@ module Mihari
         Enrichers::Whois => %i[
           enrich_whois
         ],
-        Enrichers::IPInfo => %i[
+        Enrichers::MMDB => %i[
           enrich_autonomous_system
           enrich_geolocation
         ],
@@ -208,69 +198,43 @@ module Mihari
       end
 
       class << self
-        #
-        # Search artifacts
-        #
-        # @param [Mihari::Structs::Filters::Artifact::SearchFilterWithPagination] filter
-        #
-        # @return [Array<Artifact>]
-        #
-        def search(filter)
-          limit = filter.limit.to_i
-          raise ArgumentError, "limit should be bigger than zero" unless limit.positive?
-
-          page = filter.page.to_i
-          raise ArgumentError, "page should be bigger than zero" unless page.positive?
-
-          offset = (page - 1) * limit
-
-          relation = build_relation(filter.without_pagination)
-          relation.limit(limit).offset(offset).order(id: :desc)
-        end
-
-        #
-        # Count artifacts
-        #
-        # @param [Mihari::Structs::Filters::Artifact::SearchFilter] filter
-        #
-        # @return [Integer]
-        #
-        def count(filter)
-          relation = build_relation(filter)
-          relation.distinct("artifacts.id").count
-        end
+        # @!method search_by_filter(filter)
+        #   @param [Mihari::Structs::Filters::Search] filter
+        #   @return [Array<Mihari::Models::Alert>]
 
-        #
-        # @param [Mihari::Structs::Filters::Artifact::SearchFilter] filter
-        #
-        # @return [Mihari::Models::Artifact]
-        #
-        def build_relation(filter)
-          relation = eager_load(alert: :tags)
+        # @!method count_by_filter(filter)
+        #   @param [Mihari::Structs::Filters::Search] filter
+        #   @return [Integer]
+      end
 
-          relation = relation.where(alert: { rule_id: filter.rule_id }) if filter.rule_id
-          relation = relation.where(alert: { tags: { name: filter.tag } }) if filter.tag
-          relation = relation.where("artifacts.created_at >= ?", filter.from_at) if filter.from_at
-          relation = relation.where("artifacts.created_at <= ?", filter.to_at) if filter.to_at
+      private
 
-          relation
-        end
+      def set_data_type
+        self.data_type = DataType.type(data)
       end
 
-      private
+      def set_rule_id
+        @set_rule_id ||= nil
+      end
 
-      def ipinfo
-        @ipinfo ||= Enrichers::IPInfo.new
+      def mmdb
+        @mmdb ||= Enrichers::MMDB.new
       end
 
       def shodan
         @shodan ||= Enrichers::Shodan.new
       end
 
-      def normalize_as_domain(url_or_domain)
-        return url_or_domain if data_type == "domain"
-
-        Addressable::URI.parse(url_or_domain).host
+      #
+      # @return [String, nil]
+      #
+      def domain
+        case data_type
+        when "domain"
+          data
+        when "url"
+          Addressable::URI.parse(data).host
+        end
       end
 
       def can_enrich_whois?

data/lib/mihari/models/autonomous_system.rb

@@ -7,30 +7,6 @@ module Mihari
     #
     class AutonomousSystem < ActiveRecord::Base
       belongs_to :artifact
-
-      class << self
-        include Dry::Monads[:result]
-
-        #
-        # Build AS
-        #
-        # @param [String] ip
-        # @param [Mihari::Enrichers::IPInfo] enricher
-        #
-        # @return [Mihari::AutonomousSystem, nil]
-        #
-        def build_by_ip(ip, enricher: Enrichers::IPInfo.new)
-          result = enricher.result(ip).bind do |res|
-            value = res&.asn
-            if value.nil?
-              Success nil
-            else
-              Success new(asn: value)
-            end
-          end
-          result.value_or nil
-        end
-      end
     end
   end
 end

data/lib/mihari/models/concerns/searchable.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Models
+    module Concerns
+      module Searchable
+        extend ActiveSupport::Concern
+
+        class_methods do
+          #
+          # @param [Mihari::Structs::Filters::Search] filter
+          #
+          # @return [Array<Mihari::Models::Rule>]
+          #
+          def search_by_filter(filter)
+            limit = filter.limit.to_i
+            raise ArgumentError, "limit should be greater than or equal to zero" if limit.negative?
+
+            page = filter.page.to_i
+            raise ArgumentError, "page should be greater than zero" unless page.positive?
+
+            offset = (page - 1) * limit
+
+            relation = build_relation(filter)
+            relation.limit(limit).offset(offset).order(created_at: :desc)
+          end
+
+          #
+          # @param [Mihari::Structs::Filters::Search] filter
+          #
+          # @return [Array<Mihari::Models::Rule>]
+          #
+          def count_by_filter(filter)
+            relation = build_relation(filter)
+            relation.distinct(:id).count
+          end
+
+          private
+
+          #
+          # @param [Mihari::Structs::Filters::Search] filter
+          #
+          def build_relation(filter)
+            filter.q.empty? ? all : search(filter.q)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mihari/models/cpe.rb

@@ -7,29 +7,6 @@ module Mihari
     #
     class CPE < ActiveRecord::Base
       belongs_to :artifact
-
-      class << self
-        include Dry::Monads[:result]
-
-        #
-        # Build CPEs
-        #
-        # @param [String] ip
-        # @param [Mihari::Enrichers::Shodan] enricher
-        #
-        # @return [Array<Mihari::CPE>]
-        #
-        def build_by_ip(ip, enricher: Enrichers::Shodan.new)
-          result = enricher.result(ip).bind do |res|
-            if res.nil?
-              Success []
-            else
-              Success(res.cpes.map { |cpe| new(cpe: cpe) })
-            end
-          end
-          result.value_or []
-        end
-      end
     end
   end
 end

data/lib/mihari/models/dns.rb

@@ -7,26 +7,6 @@ module Mihari
     #
     class DnsRecord < ActiveRecord::Base
       belongs_to :artifact
-
-      class << self
-        include Dry::Monads[:result]
-
-        #
-        # Build DNS records
-        #
-        # @param [String] domain
-        # @param [Mihari::Enrichers::Shodan] enricher
-        #
-        # @return [Array<Mihari::Models::DnsRecord>]
-        #
-        def build_by_domain(domain, enricher: Enrichers::GooglePublicDNS.new)
-          enricher.result(domain).bind do |res|
-            Success(
-              res.answers.map { |answer| new(resource: answer.resource_type, value: answer.data) }
-            )
-          end.value_or([])
-        end
-      end
     end
   end
 end

data/lib/mihari/models/geolocation.rb

@@ -9,30 +9,6 @@ module Mihari
     #
     class Geolocation < ActiveRecord::Base
       belongs_to :artifact
-
-      class << self
-        include Dry::Monads[:result]
-
-        #
-        # Build Geolocation
-        #
-        # @param [String] ip
-        # @param [Mihari::Enrichers::IPinfo] enricher
-        #
-        # @return [Mihari::Geolocation, nil]
-        #
-        def build_by_ip(ip, enricher: Enrichers::IPInfo.new)
-          result = enricher.result(ip).bind do |res|
-            value = res&.country_code
-            if value.nil?
-              Success nil
-            else
-              Success new(country: NormalizeCountry(value, to: :short), country_code: value)
-            end
-          end
-          result.value_or nil
-        end
-      end
     end
   end
 end

data/lib/mihari/models/port.rb

@@ -9,8 +9,6 @@ module Mihari
       belongs_to :artifact
 
       class << self
-        include Dry::Monads[:result]
-
         #
         # Build ports
         #
@@ -20,14 +18,9 @@ module Mihari
         # @return [Array<Mihari::Port>]
         #
         def build_by_ip(ip, enricher: Enrichers::Shodan.new)
-          result = enricher.result(ip).bind do |res|
-            if res.nil?
-              Success []
-            else
-              Success(res.ports.map { |port| new(port: port) })
-            end
-          end
-          result.value_or []
+          enricher.result(ip).fmap do |res|
+            (res&.ports || []).map { |port| new(port: port) }
+          end.value_or []
         end
       end
     end

data/lib/mihari/models/reverse_dns.rb

@@ -7,29 +7,6 @@ module Mihari
     #
     class ReverseDnsName < ActiveRecord::Base
       belongs_to :artifact
-
-      class << self
-        include Dry::Monads[:result]
-
-        #
-        # Build reverse DNS names
-        #
-        # @param [String] ip
-        # @param [Mihari::Enrichers::Shodan] enricher
-        #
-        # @return [Array<Mihari::Models::ReverseDnsName>]
-        #
-        def build_by_ip(ip, enricher: Enrichers::Shodan.new)
-          result = enricher.result(ip).bind do |res|
-            if res.nil?
-              Success []
-            else
-              Success(res.hostnames.map { |name| new(name: name) })
-            end
-          end
-          result.value_or []
-        end
-      end
     end
   end
 end

data/lib/mihari/models/rule.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "yaml"
-
 module Mihari
   module Models
     #
@@ -9,6 +7,16 @@ module Mihari
     #
     class Rule < ActiveRecord::Base
       has_many :alerts, dependent: :destroy
+      has_many :taggings, dependent: :destroy
+      has_many :tags, through: :taggings
+
+      include SearchCop
+      include Concerns::Searchable
+
+      search_scope :search do
+        attributes :id, :title, :description, :created_at, :updated_at
+        attributes tag: "tags.name"
+      end
 
       def symbolized_data
         @symbolized_data ||= data.deep_symbolize_keys
@@ -18,63 +26,14 @@ module Mihari
         data.to_yaml
       end
 
-      def tags
-        (data["tags"] || []).map { |tag| { name: tag } }
-      end
-
       class << self
-        #
-        # Search rules
-        #
-        # @param [Mihari::Structs::Filters::Rule::SearchFilterWithPagination] filter
-        #
-        # @return [Array<Rule>]
-        #
-        def search(filter)
-          limit = filter.limit.to_i
-          raise ArgumentError, "limit should be bigger than zero" unless limit.positive?
-
-          page = filter.page.to_i
-          raise ArgumentError, "page should be bigger than zero" unless page.positive?
-
-          offset = (page - 1) * limit
-
-          relation = build_relation(filter.without_pagination)
-          relation.limit(limit).offset(offset).order(created_at: :desc)
-        end
-
-        #
-        # Count alerts
-        #
-        # @param [Mihari::Structs::Filters::Rule::SearchFilterWithPagination] filter
-        #
-        # @return [Integer]
-        #
-        def count(filter)
-          relation = build_relation(filter)
-          relation.distinct("rules.id").count
-        end
-
-        private
-
-        #
-        # @param [Mihari::Structs::Filters::Rule::SearchFilter] filter
-        #
-        # @return [Mihari::Models::Rule]
-        #
-        def build_relation(filter)
-          relation = includes(alerts: :tags)
-
-          relation = relation.where(alerts: { tags: { name: filter.tag } }) if filter.tag
-
-          relation = relation.where("rules.title LIKE ?", "%#{filter.title}%") if filter.title
-          relation = relation.where("rules.description LIKE ?", "%#{filter.description}%") if filter.description
-
-          relation = relation.where("rules.created_at >= ?", filter.from_at) if filter.from_at
-          relation = relation.where("rules.created_at <= ?", filter.to_at) if filter.to_at
+        # @!method search_by_filter(filter)
+        #   @param [Mihari::Structs::Filters::Search] filter
+        #   @return [Array<Mihari::Models::Alert>]
 
-          relation
-        end
+        # @!method count_by_filter(filter)
+        #   @param [Mihari::Structs::Filters::Search] filter
+        #   @return [Integer]
       end
     end
   end