mihari 6.2.0 → 7.0.0

data/lib/mihari/models/alert.rb

@@ -6,86 +6,31 @@ module Mihari
     # Alert model
     #
     class Alert < ActiveRecord::Base
-      has_many :taggings, dependent: :destroy
-      has_many :artifacts, dependent: :destroy
-      has_many :tags, through: :taggings
-
       belongs_to :rule
 
-      class << self
-        #
-        # Search alerts
-        #
-        # @param [Mihari::Structs::Filters::Alert::SearchFilterWithPagination] filter
-        #
-        # @return [Array<Alert>]
-        #
-        def search(filter)
-          limit = filter.limit.to_i
-          raise ArgumentError, "limit should be bigger than zero" unless limit.positive?
-
-          page = filter.page.to_i
-          raise ArgumentError, "page should be bigger than zero" unless page.positive?
-
-          offset = (page - 1) * limit
-
-          relation = build_relation(filter.without_pagination)
-          alert_ids = relation.limit(limit).offset(offset).order(id: :desc).pluck(:id).uniq
-          eager_load(:artifacts, :tags).where(id: [alert_ids]).order(id: :desc)
-        end
-
-        #
-        # Count alerts
-        #
-        # @param [Mihari::Structs::Filters::Alert::SearchFilter] filter
-        #
-        # @return [Integer]
-        #
-        def count(filter)
-          relation = build_relation(filter)
-          relation.distinct("alerts.id").count
-        end
-
-        private
-
-        #
-        # @param [Mihari::Structs::Filters::Alert::SearchFilter] filter
-        #
-        # @return [Array<Integer>]
-        #
-        def get_artifact_ids_by_filter(filter)
-          artifact_ids = []
-
-          if filter.artifact_data
-            artifact = Artifact.where(data: filter.artifact_data)
-            artifact_ids = artifact.pluck(:id)
-            # set invalid ID if nothing is matched with the filters
-            artifact_ids = [-1] if artifact_ids.empty?
-          end
-
-          artifact_ids
-        end
-
-        #
-        # @param [Mihari::Structs::Filters::Alert::SearchFilter] filter
-        #
-        # @return [Mihari::Models::Alert]
-        #
-        def build_relation(filter)
-          artifact_ids = get_artifact_ids_by_filter(filter)
-
-          relation = includes(:artifacts, :tags)
-
-          relation = relation.where(artifacts: { id: artifact_ids }) unless artifact_ids.empty?
-          relation = relation.where(tags: { name: filter.tag_name }) if filter.tag_name
-
-          relation = relation.where(rule_id: filter.rule_id) if filter.rule_id
+      has_many :artifacts, dependent: :destroy
+      has_many :tags, through: :rule
+
+      include SearchCop
+      include Concerns::Searchable
+
+      search_scope :search do
+        attributes :id, :created_at, "rule.id", "rule.title", "rule.description"
+        attributes "artifact.data" => "artifacts.data"
+        attributes "artifact.data_type" => "artifacts.data_type"
+        attributes "artifact.source" => "artifacts.source"
+        attributes "artifact.query" => "artifacts.query"
+        attributes tag: "tags.name"
+      end
 
-          relation = relation.where("alerts.created_at >= ?", filter.from_at) if filter.from_at
-          relation = relation.where("alerts.created_at <= ?", filter.to_at) if filter.to_at
+      class << self
+        # @!method search_by_filter(filter)
+        #   @param [Mihari::Structs::Filters::Search] filter
+        #   @return [Array<Mihari::Models::Alert>]
 
-          relation
-        end
+        # @!method count_by_filter(filter)
+        #   @param [Mihari::Structs::Filters::Search] filter
+        #   @return [Integer]
       end
     end
   end

data/lib/mihari/models/artifact.rb

@@ -22,45 +22,47 @@ module Mihari
       has_one :autonomous_system, dependent: :destroy
       has_one :geolocation, dependent: :destroy
       has_one :whois_record, dependent: :destroy
+      has_one :rule, through: :alert
 
       has_many :cpes, dependent: :destroy
       has_many :dns_records, dependent: :destroy
       has_many :ports, dependent: :destroy
       has_many :reverse_dns_names, dependent: :destroy
+      has_many :tags, through: :alert
 
       include ActiveModel::Validations
+      include SearchCop
+      include Concerns::Searchable
+
+      search_scope :search do
+        attributes :id, :data, :data_type, :source, :query, :created_at, "alert.id", "rule.id", "rule.title",
+          "rule.description"
+        attributes tag: "tags.name"
+        attributes asn: "autonomous_system.asn"
+        attributes country_code: "geolocation.country_code"
+        attributes "dns_record.value": "dns_records.value"
+        attributes "dns_record.resource": "dns_records.resource"
+        attributes reverse_dns_name: "reverse_dns_names.name"
+        attributes cpe: "cpes.name"
+        attributes port: "ports.port"
+      end
 
       validates_with ArtifactValidator
 
-      # @return [Array<Mihari::Tag>] Tags
-      attr_accessor :tags
+      after_initialize :set_data_type, :set_rule_id, if: :new_record?
 
-      # @return [String, nil] Rule ID
+      # @return [String, nil]
       attr_accessor :rule_id
 
-      def initialize(*args, **kwargs)
-        attrs = args.first || kwargs
-        data_ = attrs[:data]
-
-        raise TypeError if data_.is_a?(Array) || data_.is_a?(Hash)
-
-        super(*args, **kwargs)
-
-        self.data_type = DataType.type(data)
-
-        @tags = []
-        @rule_id = ""
-      end
-
       #
-      # Check uniqueness of artifact
+      # Check uniqueness
       #
       # @param [Time, nil] base_time Base time to check decaying
-      # @param [Integer, nil] artifact_lifetime Artifact lifetime (TTL) in seconds
+      # @param [Integer, nil] artifact_ttl Artifact TTL in seconds
       #
       # @return [Boolean] true if it is unique. Otherwise false.
       #
-      def unique?(base_time: nil, artifact_lifetime: nil)
+      def unique?(base_time: nil, artifact_ttl: nil)
         artifact = self.class.joins(:alert).where(
           data: data,
           alert: { rule_id: rule_id }
@@ -68,12 +70,12 @@ module Mihari
         return true if artifact.nil?
 
         # check whether the artifact is decayed or not
-        return false if artifact_lifetime.nil?
+        return false if artifact_ttl.nil?
 
         # use the current UTC time if base_time is not given (for testing)
         base_time ||= Time.now.utc
 
-        decayed_at = base_time - (artifact_lifetime || -1).seconds
+        decayed_at = base_time - (artifact_ttl || -1).seconds
         artifact.created_at < decayed_at
       end
 
@@ -105,7 +107,7 @@ module Mihari
       # @param [Mihari::Enrichers::Shodan] enricher
       #
       def enrich_reverse_dns(enricher = Enrichers::Shodan.new)
-        return unless can_enrich_revese_dns?
+        return unless can_enrich_reverse_dns?
 
         self.reverse_dns_names = ReverseDnsName.build_by_ip(data, enricher: enricher)
       end
@@ -115,7 +117,7 @@ module Mihari
       #
       # @param [Mihari::Enrichers::IPInfo] enricher
       #
-      def enrich_geolocation(enricher = Enrichers::IPInfo.new)
+      def enrich_geolocation(enricher = Enrichers::MMDB.new)
         return unless can_enrich_geolocation?
 
         self.geolocation = Geolocation.build_by_ip(data, enricher: enricher)
@@ -126,7 +128,7 @@ module Mihari
       #
       # @param [Mihari::Enrichers::IPInfo] enricher
       #
-      def enrich_autonomous_system(enricher = Enrichers::IPInfo.new)
+      def enrich_autonomous_system(enricher = Enrichers::MMDB.new)
         return unless can_enrich_autonomous_system?
 
         self.autonomous_system = AutonomousSystem.build_by_ip(data, enricher: enricher)
@@ -158,9 +160,9 @@ module Mihari
       # Enrich all the enrichable relationships of the artifact
       #
       def enrich_all
-        enrich_autonomous_system ipinfo
+        enrich_autonomous_system mmdb
         enrich_dns
-        enrich_geolocation ipinfo
+        enrich_geolocation mmdb
         enrich_reverse_dns shodan
         enrich_whois
         enrich_ports shodan
@@ -171,7 +173,7 @@ module Mihari
         Enrichers::Whois => %i[
           enrich_whois
         ],
-        Enrichers::IPInfo => %i[
+        Enrichers::MMDB => %i[
           enrich_autonomous_system
           enrich_geolocation
         ],
@@ -195,10 +197,28 @@ module Mihari
         methods.each { |method| send(method, enricher) if respond_to?(method) }
       end
 
+      class << self
+        # @!method search_by_filter(filter)
+        #   @param [Mihari::Structs::Filters::Search] filter
+        #   @return [Array<Mihari::Models::Alert>]
+
+        # @!method count_by_filter(filter)
+        #   @param [Mihari::Structs::Filters::Search] filter
+        #   @return [Integer]
+      end
+
       private
 
-      def ipinfo
-        @ipinfo ||= Enrichers::IPInfo.new
+      def set_data_type
+        self.data_type = DataType.type(data)
+      end
+
+      def set_rule_id
+        @set_rule_id ||= nil
+      end
+
+      def mmdb
+        @mmdb ||= Enrichers::MMDB.new
       end
 
       def shodan
@@ -219,7 +239,7 @@ module Mihari
         %w[domain url].include?(data_type) && dns_records.empty?
       end
 
-      def can_enrich_revese_dns?
+      def can_enrich_reverse_dns?
         data_type == "ip" && reverse_dns_names.empty?
       end
 

data/lib/mihari/models/autonomous_system.rb

@@ -9,26 +9,18 @@ module Mihari
       belongs_to :artifact
 
       class << self
-        include Dry::Monads[:result]
-
         #
         # Build AS
         #
         # @param [String] ip
-        # @param [Mihari::Enrichers::IPInfo] enricher
+        # @param [Mihari::Enrichers::MMDB] enricher
         #
         # @return [Mihari::AutonomousSystem, nil]
         #
-        def build_by_ip(ip, enricher: Enrichers::IPInfo.new)
-          result = enricher.result(ip).bind do |res|
-            value = res&.asn
-            if value.nil?
-              Success nil
-            else
-              Success new(asn: value)
-            end
-          end
-          result.value_or nil
+        def build_by_ip(ip, enricher: Enrichers::MMDB.new)
+          enricher.result(ip).fmap do |res|
+            new(asn: res.asn) if res.asn
+          end.value_or nil
         end
       end
     end

data/lib/mihari/models/concerns/searchable.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Models
+    module Concerns
+      module Searchable
+        extend ActiveSupport::Concern
+
+        class_methods do
+          #
+          # @param [Mihari::Structs::Filters::Search] filter
+          #
+          # @return [Array<Mihari::Models::Rule>]
+          #
+          def search_by_filter(filter)
+            limit = filter.limit.to_i
+            raise ArgumentError, "limit should be greater than or equal to zero" if limit.negative?
+
+            page = filter.page.to_i
+            raise ArgumentError, "page should be greater than zero" unless page.positive?
+
+            offset = (page - 1) * limit
+
+            relation = build_relation(filter)
+            relation.limit(limit).offset(offset).order(created_at: :desc)
+          end
+
+          #
+          # @param [Mihari::Structs::Filters::Search] filter
+          #
+          # @return [Array<Mihari::Models::Rule>]
+          #
+          def count_by_filter(filter)
+            relation = build_relation(filter)
+            relation.distinct(:id).count
+          end
+
+          private
+
+          #
+          # @param [Mihari::Structs::Filters::Search] filter
+          #
+          def build_relation(filter)
+            filter.q.empty? ? all : search(filter.q)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mihari/models/cpe.rb

@@ -9,8 +9,6 @@ module Mihari
       belongs_to :artifact
 
       class << self
-        include Dry::Monads[:result]
-
         #
         # Build CPEs
         #
@@ -20,14 +18,9 @@ module Mihari
         # @return [Array<Mihari::CPE>]
         #
         def build_by_ip(ip, enricher: Enrichers::Shodan.new)
-          result = enricher.result(ip).bind do |res|
-            if res.nil?
-              Success []
-            else
-              Success(res.cpes.map { |cpe| new(cpe: cpe) })
-            end
-          end
-          result.value_or []
+          enricher.result(ip).fmap do |res|
+            (res&.cpes || []).map { |cpe| new(cpe: cpe) }
+          end.value_or []
         end
       end
     end

data/lib/mihari/models/dns.rb

@@ -9,8 +9,6 @@ module Mihari
       belongs_to :artifact
 
       class << self
-        include Dry::Monads[:result]
-
         #
         # Build DNS records
         #
@@ -20,10 +18,8 @@ module Mihari
         # @return [Array<Mihari::Models::DnsRecord>]
         #
         def build_by_domain(domain, enricher: Enrichers::GooglePublicDNS.new)
-          enricher.result(domain).bind do |res|
-            Success(
-              res.answers.map { |answer| new(resource: answer.resource_type, value: answer.data) }
-            )
+          enricher.result(domain).fmap do |res|
+            res.answers.map { |answer| new(resource: answer.resource_type, value: answer.data) }
           end.value_or([])
         end
       end

data/lib/mihari/models/geolocation.rb

@@ -11,26 +11,21 @@ module Mihari
       belongs_to :artifact
 
       class << self
-        include Dry::Monads[:result]
-
         #
         # Build Geolocation
         #
         # @param [String] ip
-        # @param [Mihari::Enrichers::IPinfo] enricher
+        # @param [Mihari::Enrichers::MMDB] enricher
         #
         # @return [Mihari::Geolocation, nil]
         #
-        def build_by_ip(ip, enricher: Enrichers::IPInfo.new)
-          result = enricher.result(ip).bind do |res|
-            value = res&.country_code
-            if value.nil?
-              Success nil
-            else
-              Success new(country: NormalizeCountry(value, to: :short), country_code: value)
+        def build_by_ip(ip, enricher: Enrichers::MMDB.new)
+          enricher.result(ip).fmap do |res|
+            if res.country_code
+              new(country: NormalizeCountry(res.country_code, to: :short),
+                country_code: res.country_code)
             end
-          end
-          result.value_or nil
+          end.value_or nil
         end
       end
     end

data/lib/mihari/models/port.rb

@@ -9,8 +9,6 @@ module Mihari
       belongs_to :artifact
 
       class << self
-        include Dry::Monads[:result]
-
         #
         # Build ports
         #
@@ -20,14 +18,9 @@ module Mihari
         # @return [Array<Mihari::Port>]
         #
         def build_by_ip(ip, enricher: Enrichers::Shodan.new)
-          result = enricher.result(ip).bind do |res|
-            if res.nil?
-              Success []
-            else
-              Success(res.ports.map { |port| new(port: port) })
-            end
-          end
-          result.value_or []
+          enricher.result(ip).fmap do |res|
+            (res&.ports || []).map { |port| new(port: port) }
+          end.value_or []
         end
       end
     end

data/lib/mihari/models/reverse_dns.rb

@@ -20,14 +20,9 @@ module Mihari
         # @return [Array<Mihari::Models::ReverseDnsName>]
         #
         def build_by_ip(ip, enricher: Enrichers::Shodan.new)
-          result = enricher.result(ip).bind do |res|
-            if res.nil?
-              Success []
-            else
-              Success(res.hostnames.map { |name| new(name: name) })
-            end
-          end
-          result.value_or []
+          enricher.result(ip).fmap do |res|
+            (res&.hostnames || []).map { |name| new(name: name) }
+          end.value_or []
         end
       end
     end

data/lib/mihari/models/rule.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "yaml"
-
 module Mihari
   module Models
     #
@@ -9,6 +7,16 @@ module Mihari
     #
     class Rule < ActiveRecord::Base
       has_many :alerts, dependent: :destroy
+      has_many :taggings, dependent: :destroy
+      has_many :tags, through: :taggings
+
+      include SearchCop
+      include Concerns::Searchable
+
+      search_scope :search do
+        attributes :id, :title, :description, :created_at, :updated_at
+        attributes tag: "tags.name"
+      end
 
       def symbolized_data
         @symbolized_data ||= data.deep_symbolize_keys
@@ -18,66 +26,14 @@ module Mihari
         data.to_yaml
       end
 
-      def tags
-        (data["tags"] || []).map { |tag| { name: tag } }
-      end
-
       class << self
-        #
-        # Search rules
-        #
-        # @param [Mihari::Structs::Filters::Rule::SearchFilterWithPagination] filter
-        #
-        # @return [Array<Rule>]
-        #
-        def search(filter)
-          limit = filter.limit.to_i
-          raise ArgumentError, "limit should be bigger than zero" unless limit.positive?
-
-          page = filter.page.to_i
-          raise ArgumentError, "page should be bigger than zero" unless page.positive?
-
-          offset = (page - 1) * limit
-
-          relation = build_relation(filter.without_pagination)
-
-          # TODO: improve queires
-          rule_ids = relation.limit(limit).offset(offset).order(created_at: :desc).pluck(:id).uniq
-          where(id: [rule_ids]).order(created_at: :desc)
-        end
-
-        #
-        # Count alerts
-        #
-        # @param [Mihari::Structs::Filters::Rule::SearchFilterWithPagination] filter
-        #
-        # @return [Integer]
-        #
-        def count(filter)
-          relation = build_relation(filter)
-          relation.distinct("rules.id").count
-        end
-
-        private
-
-        #
-        # @param [Mihari::Structs::Filters::Rule::SearchFilter] filter
-        #
-        # @return [Mihari::Models::Rule]
-        #
-        def build_relation(filter)
-          relation = includes(alerts: :tags)
-
-          relation = relation.where(alerts: { tags: { name: filter.tag_name } }) if filter.tag_name
-
-          relation = relation.where("rules.title LIKE ?", "%#{filter.title}%") if filter.title
-          relation = relation.where("rules.description LIKE ?", "%#{filter.description}%") if filter.description
-
-          relation = relation.where("rules.created_at >= ?", filter.from_at) if filter.from_at
-          relation = relation.where("rules.created_at <= ?", filter.to_at) if filter.to_at
+        # @!method search_by_filter(filter)
+        #   @param [Mihari::Structs::Filters::Search] filter
+        #   @return [Array<Mihari::Models::Alert>]
 
-          relation
-        end
+        # @!method count_by_filter(filter)
+        #   @param [Mihari::Structs::Filters::Search] filter
+        #   @return [Integer]
       end
     end
   end

data/lib/mihari/models/tag.rb

@@ -7,7 +7,23 @@ module Mihari
     #
     class Tag < ActiveRecord::Base
       has_many :taggings, dependent: :destroy
-      has_many :tags, through: :taggings
+
+      include SearchCop
+      include Concerns::Searchable
+
+      search_scope :search do
+        attributes :id, :name
+      end
+
+      class << self
+        # @!method search_by_filter(filter)
+        #   @param [Mihari::Structs::Filters::Search] filter
+        #   @return [Array<Mihari::Models::Tag>]
+
+        # @!method count_by_filter(filter)
+        #   @param [Mihari::Structs::Filters::Search] filter
+        #   @return [Integer]
+      end
     end
   end
 end

data/lib/mihari/models/tagging.rb

@@ -6,7 +6,7 @@ module Mihari
     # Tagging model
     #
     class Tagging < ActiveRecord::Base
-      belongs_to :alert
+      belongs_to :rule
       belongs_to :tag
     end
   end

data/lib/mihari/models/whois.rb

@@ -8,8 +8,6 @@ module Mihari
     class WhoisRecord < ActiveRecord::Base
       belongs_to :artifact
 
-      @memo = {}
-
       class << self
         #
         # Build whois record
@@ -20,8 +18,7 @@ module Mihari
         # @return [WhoisRecord, nil]
         #
         def build_by_domain(domain, enricher: Enrichers::Whois.new)
-          result = enricher.result(domain)
-          result.value_or nil
+          enricher.result(domain).value_or nil
         end
       end
     end