mihari 6.3.0 → 7.0.0

data/lib/mihari/models/artifact.rb

@@ -22,45 +22,47 @@ module Mihari
       has_one :autonomous_system, dependent: :destroy
       has_one :geolocation, dependent: :destroy
       has_one :whois_record, dependent: :destroy
+      has_one :rule, through: :alert
 
       has_many :cpes, dependent: :destroy
       has_many :dns_records, dependent: :destroy
       has_many :ports, dependent: :destroy
       has_many :reverse_dns_names, dependent: :destroy
+      has_many :tags, through: :alert
 
       include ActiveModel::Validations
+      include SearchCop
+      include Concerns::Searchable
+
+      search_scope :search do
+        attributes :id, :data, :data_type, :source, :query, :created_at, "alert.id", "rule.id", "rule.title",
+          "rule.description"
+        attributes tag: "tags.name"
+        attributes asn: "autonomous_system.asn"
+        attributes country_code: "geolocation.country_code"
+        attributes "dns_record.value": "dns_records.value"
+        attributes "dns_record.resource": "dns_records.resource"
+        attributes reverse_dns_name: "reverse_dns_names.name"
+        attributes cpe: "cpes.name"
+        attributes port: "ports.port"
+      end
 
       validates_with ArtifactValidator
 
-      # @return [Array<Mihari::Tag>] Tags
-      attr_accessor :tags
+      after_initialize :set_data_type, :set_rule_id, if: :new_record?
 
-      # @return [String, nil] Rule ID
+      # @return [String, nil]
       attr_accessor :rule_id
 
-      def initialize(*args, **kwargs)
-        attrs = args.first || kwargs
-        data_ = attrs[:data]
-
-        raise TypeError if data_.is_a?(Array) || data_.is_a?(Hash)
-
-        super(*args, **kwargs)
-
-        self.data_type = DataType.type(data)
-
-        @tags = []
-        @rule_id = ""
-      end
-
       #
-      # Check uniqueness of artifact
+      # Check uniqueness
       #
       # @param [Time, nil] base_time Base time to check decaying
-      # @param [Integer, nil] artifact_lifetime Artifact lifetime (TTL) in seconds
+      # @param [Integer, nil] artifact_ttl Artifact TTL in seconds
       #
       # @return [Boolean] true if it is unique. Otherwise false.
       #
-      def unique?(base_time: nil, artifact_lifetime: nil)
+      def unique?(base_time: nil, artifact_ttl: nil)
         artifact = self.class.joins(:alert).where(
           data: data,
           alert: { rule_id: rule_id }
@@ -68,27 +70,15 @@ module Mihari
         return true if artifact.nil?
 
         # check whether the artifact is decayed or not
-        return false if artifact_lifetime.nil?
+        return false if artifact_ttl.nil?
 
         # use the current UTC time if base_time is not given (for testing)
         base_time ||= Time.now.utc
 
-        decayed_at = base_time - (artifact_lifetime || -1).seconds
+        decayed_at = base_time - (artifact_ttl || -1).seconds
         artifact.created_at < decayed_at
       end
 
-      #
-      # Count artifacts
-      #
-      # @param [Mihari::Structs::Filters::Artifact::SearchFilter] filter
-      #
-      # @return [Integer]
-      #
-      def count(filter)
-        relation = build_relation(filter)
-        relation.distinct("artifact.id").count
-      end
-
       #
       # Enrich whois record
       #
@@ -127,7 +117,7 @@ module Mihari
       #
       # @param [Mihari::Enrichers::IPInfo] enricher
       #
-      def enrich_geolocation(enricher = Enrichers::IPInfo.new)
+      def enrich_geolocation(enricher = Enrichers::MMDB.new)
         return unless can_enrich_geolocation?
 
         self.geolocation = Geolocation.build_by_ip(data, enricher: enricher)
@@ -138,7 +128,7 @@ module Mihari
       #
       # @param [Mihari::Enrichers::IPInfo] enricher
       #
-      def enrich_autonomous_system(enricher = Enrichers::IPInfo.new)
+      def enrich_autonomous_system(enricher = Enrichers::MMDB.new)
         return unless can_enrich_autonomous_system?
 
         self.autonomous_system = AutonomousSystem.build_by_ip(data, enricher: enricher)
@@ -170,9 +160,9 @@ module Mihari
       # Enrich all the enrichable relationships of the artifact
       #
       def enrich_all
-        enrich_autonomous_system ipinfo
+        enrich_autonomous_system mmdb
         enrich_dns
-        enrich_geolocation ipinfo
+        enrich_geolocation mmdb
         enrich_reverse_dns shodan
         enrich_whois
         enrich_ports shodan
@@ -183,7 +173,7 @@ module Mihari
         Enrichers::Whois => %i[
           enrich_whois
         ],
-        Enrichers::IPInfo => %i[
+        Enrichers::MMDB => %i[
           enrich_autonomous_system
           enrich_geolocation
         ],
@@ -208,59 +198,27 @@ module Mihari
       end
 
       class << self
-        #
-        # Search artifacts
-        #
-        # @param [Mihari::Structs::Filters::Artifact::SearchFilterWithPagination] filter
-        #
-        # @return [Array<Artifact>]
-        #
-        def search(filter)
-          limit = filter.limit.to_i
-          raise ArgumentError, "limit should be bigger than zero" unless limit.positive?
-
-          page = filter.page.to_i
-          raise ArgumentError, "page should be bigger than zero" unless page.positive?
-
-          offset = (page - 1) * limit
-
-          relation = build_relation(filter.without_pagination)
-          relation.limit(limit).offset(offset).order(id: :desc)
-        end
-
-        #
-        # Count artifacts
-        #
-        # @param [Mihari::Structs::Filters::Artifact::SearchFilter] filter
-        #
-        # @return [Integer]
-        #
-        def count(filter)
-          relation = build_relation(filter)
-          relation.distinct("artifacts.id").count
-        end
-
-        #
-        # @param [Mihari::Structs::Filters::Artifact::SearchFilter] filter
-        #
-        # @return [Mihari::Models::Artifact]
-        #
-        def build_relation(filter)
-          relation = eager_load(alert: :tags)
-
-          relation = relation.where(alert: { rule_id: filter.rule_id }) if filter.rule_id
-          relation = relation.where(alert: { tags: { name: filter.tag } }) if filter.tag
-          relation = relation.where("artifacts.created_at >= ?", filter.from_at) if filter.from_at
-          relation = relation.where("artifacts.created_at <= ?", filter.to_at) if filter.to_at
-
-          relation
-        end
+        # @!method search_by_filter(filter)
+        #   @param [Mihari::Structs::Filters::Search] filter
+        #   @return [Array<Mihari::Models::Alert>]
+
+        # @!method count_by_filter(filter)
+        #   @param [Mihari::Structs::Filters::Search] filter
+        #   @return [Integer]
       end
 
       private
 
-      def ipinfo
-        @ipinfo ||= Enrichers::IPInfo.new
+      def set_data_type
+        self.data_type = DataType.type(data)
+      end
+
+      def set_rule_id
+        @set_rule_id ||= nil
+      end
+
+      def mmdb
+        @mmdb ||= Enrichers::MMDB.new
       end
 
       def shodan

data/lib/mihari/models/autonomous_system.rb

@@ -9,26 +9,18 @@ module Mihari
       belongs_to :artifact
 
       class << self
-        include Dry::Monads[:result]
-
         #
         # Build AS
         #
         # @param [String] ip
-        # @param [Mihari::Enrichers::IPInfo] enricher
+        # @param [Mihari::Enrichers::MMDB] enricher
         #
         # @return [Mihari::AutonomousSystem, nil]
         #
-        def build_by_ip(ip, enricher: Enrichers::IPInfo.new)
-          result = enricher.result(ip).bind do |res|
-            value = res&.asn
-            if value.nil?
-              Success nil
-            else
-              Success new(asn: value)
-            end
-          end
-          result.value_or nil
+        def build_by_ip(ip, enricher: Enrichers::MMDB.new)
+          enricher.result(ip).fmap do |res|
+            new(asn: res.asn) if res.asn
+          end.value_or nil
         end
       end
     end

data/lib/mihari/models/concerns/searchable.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Models
+    module Concerns
+      module Searchable
+        extend ActiveSupport::Concern
+
+        class_methods do
+          #
+          # @param [Mihari::Structs::Filters::Search] filter
+          #
+          # @return [Array<Mihari::Models::Rule>]
+          #
+          def search_by_filter(filter)
+            limit = filter.limit.to_i
+            raise ArgumentError, "limit should be greater than or equal to zero" if limit.negative?
+
+            page = filter.page.to_i
+            raise ArgumentError, "page should be greater than zero" unless page.positive?
+
+            offset = (page - 1) * limit
+
+            relation = build_relation(filter)
+            relation.limit(limit).offset(offset).order(created_at: :desc)
+          end
+
+          #
+          # @param [Mihari::Structs::Filters::Search] filter
+          #
+          # @return [Array<Mihari::Models::Rule>]
+          #
+          def count_by_filter(filter)
+            relation = build_relation(filter)
+            relation.distinct(:id).count
+          end
+
+          private
+
+          #
+          # @param [Mihari::Structs::Filters::Search] filter
+          #
+          def build_relation(filter)
+            filter.q.empty? ? all : search(filter.q)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mihari/models/cpe.rb

@@ -9,8 +9,6 @@ module Mihari
       belongs_to :artifact
 
       class << self
-        include Dry::Monads[:result]
-
         #
         # Build CPEs
         #
@@ -20,14 +18,9 @@ module Mihari
         # @return [Array<Mihari::CPE>]
         #
         def build_by_ip(ip, enricher: Enrichers::Shodan.new)
-          result = enricher.result(ip).bind do |res|
-            if res.nil?
-              Success []
-            else
-              Success(res.cpes.map { |cpe| new(cpe: cpe) })
-            end
-          end
-          result.value_or []
+          enricher.result(ip).fmap do |res|
+            (res&.cpes || []).map { |cpe| new(cpe: cpe) }
+          end.value_or []
         end
       end
     end

data/lib/mihari/models/dns.rb

@@ -9,8 +9,6 @@ module Mihari
       belongs_to :artifact
 
       class << self
-        include Dry::Monads[:result]
-
         #
         # Build DNS records
         #
@@ -20,10 +18,8 @@ module Mihari
         # @return [Array<Mihari::Models::DnsRecord>]
         #
         def build_by_domain(domain, enricher: Enrichers::GooglePublicDNS.new)
-          enricher.result(domain).bind do |res|
-            Success(
-              res.answers.map { |answer| new(resource: answer.resource_type, value: answer.data) }
-            )
+          enricher.result(domain).fmap do |res|
+            res.answers.map { |answer| new(resource: answer.resource_type, value: answer.data) }
           end.value_or([])
         end
       end

data/lib/mihari/models/geolocation.rb

@@ -11,26 +11,21 @@ module Mihari
       belongs_to :artifact
 
       class << self
-        include Dry::Monads[:result]
-
         #
         # Build Geolocation
         #
         # @param [String] ip
-        # @param [Mihari::Enrichers::IPinfo] enricher
+        # @param [Mihari::Enrichers::MMDB] enricher
         #
         # @return [Mihari::Geolocation, nil]
         #
-        def build_by_ip(ip, enricher: Enrichers::IPInfo.new)
-          result = enricher.result(ip).bind do |res|
-            value = res&.country_code
-            if value.nil?
-              Success nil
-            else
-              Success new(country: NormalizeCountry(value, to: :short), country_code: value)
+        def build_by_ip(ip, enricher: Enrichers::MMDB.new)
+          enricher.result(ip).fmap do |res|
+            if res.country_code
+              new(country: NormalizeCountry(res.country_code, to: :short),
+                country_code: res.country_code)
             end
-          end
-          result.value_or nil
+          end.value_or nil
         end
       end
     end

data/lib/mihari/models/port.rb

@@ -9,8 +9,6 @@ module Mihari
       belongs_to :artifact
 
       class << self
-        include Dry::Monads[:result]
-
         #
         # Build ports
         #
@@ -20,14 +18,9 @@ module Mihari
         # @return [Array<Mihari::Port>]
         #
         def build_by_ip(ip, enricher: Enrichers::Shodan.new)
-          result = enricher.result(ip).bind do |res|
-            if res.nil?
-              Success []
-            else
-              Success(res.ports.map { |port| new(port: port) })
-            end
-          end
-          result.value_or []
+          enricher.result(ip).fmap do |res|
+            (res&.ports || []).map { |port| new(port: port) }
+          end.value_or []
         end
       end
     end

data/lib/mihari/models/reverse_dns.rb

@@ -20,14 +20,9 @@ module Mihari
         # @return [Array<Mihari::Models::ReverseDnsName>]
         #
         def build_by_ip(ip, enricher: Enrichers::Shodan.new)
-          result = enricher.result(ip).bind do |res|
-            if res.nil?
-              Success []
-            else
-              Success(res.hostnames.map { |name| new(name: name) })
-            end
-          end
-          result.value_or []
+          enricher.result(ip).fmap do |res|
+            (res&.hostnames || []).map { |name| new(name: name) }
+          end.value_or []
         end
       end
     end

data/lib/mihari/models/rule.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "yaml"
-
 module Mihari
   module Models
     #
@@ -9,6 +7,16 @@ module Mihari
     #
     class Rule < ActiveRecord::Base
       has_many :alerts, dependent: :destroy
+      has_many :taggings, dependent: :destroy
+      has_many :tags, through: :taggings
+
+      include SearchCop
+      include Concerns::Searchable
+
+      search_scope :search do
+        attributes :id, :title, :description, :created_at, :updated_at
+        attributes tag: "tags.name"
+      end
 
       def symbolized_data
         @symbolized_data ||= data.deep_symbolize_keys
@@ -18,63 +26,14 @@ module Mihari
         data.to_yaml
       end
 
-      def tags
-        (data["tags"] || []).map { |tag| { name: tag } }
-      end
-
       class << self
-        #
-        # Search rules
-        #
-        # @param [Mihari::Structs::Filters::Rule::SearchFilterWithPagination] filter
-        #
-        # @return [Array<Rule>]
-        #
-        def search(filter)
-          limit = filter.limit.to_i
-          raise ArgumentError, "limit should be bigger than zero" unless limit.positive?
-
-          page = filter.page.to_i
-          raise ArgumentError, "page should be bigger than zero" unless page.positive?
-
-          offset = (page - 1) * limit
-
-          relation = build_relation(filter.without_pagination)
-          relation.limit(limit).offset(offset).order(created_at: :desc)
-        end
-
-        #
-        # Count alerts
-        #
-        # @param [Mihari::Structs::Filters::Rule::SearchFilterWithPagination] filter
-        #
-        # @return [Integer]
-        #
-        def count(filter)
-          relation = build_relation(filter)
-          relation.distinct("rules.id").count
-        end
-
-        private
-
-        #
-        # @param [Mihari::Structs::Filters::Rule::SearchFilter] filter
-        #
-        # @return [Mihari::Models::Rule]
-        #
-        def build_relation(filter)
-          relation = includes(alerts: :tags)
-
-          relation = relation.where(alerts: { tags: { name: filter.tag } }) if filter.tag
-
-          relation = relation.where("rules.title LIKE ?", "%#{filter.title}%") if filter.title
-          relation = relation.where("rules.description LIKE ?", "%#{filter.description}%") if filter.description
-
-          relation = relation.where("rules.created_at >= ?", filter.from_at) if filter.from_at
-          relation = relation.where("rules.created_at <= ?", filter.to_at) if filter.to_at
+        # @!method search_by_filter(filter)
+        #   @param [Mihari::Structs::Filters::Search] filter
+        #   @return [Array<Mihari::Models::Alert>]
 
-          relation
-        end
+        # @!method count_by_filter(filter)
+        #   @param [Mihari::Structs::Filters::Search] filter
+        #   @return [Integer]
       end
     end
   end

data/lib/mihari/models/tag.rb

@@ -7,7 +7,23 @@ module Mihari
     #
     class Tag < ActiveRecord::Base
       has_many :taggings, dependent: :destroy
-      has_many :tags, through: :taggings
+
+      include SearchCop
+      include Concerns::Searchable
+
+      search_scope :search do
+        attributes :id, :name
+      end
+
+      class << self
+        # @!method search_by_filter(filter)
+        #   @param [Mihari::Structs::Filters::Search] filter
+        #   @return [Array<Mihari::Models::Tag>]
+
+        # @!method count_by_filter(filter)
+        #   @param [Mihari::Structs::Filters::Search] filter
+        #   @return [Integer]
+      end
     end
   end
 end

data/lib/mihari/models/tagging.rb

@@ -6,7 +6,7 @@ module Mihari
     # Tagging model
     #
     class Tagging < ActiveRecord::Base
-      belongs_to :alert
+      belongs_to :rule
       belongs_to :tag
     end
   end

data/lib/mihari/models/whois.rb

@@ -8,8 +8,6 @@ module Mihari
     class WhoisRecord < ActiveRecord::Base
       belongs_to :artifact
 
-      @memo = {}
-
       class << self
         #
         # Build whois record
@@ -20,8 +18,7 @@ module Mihari
         # @return [WhoisRecord, nil]
         #
         def build_by_domain(domain, enricher: Enrichers::Whois.new)
-          result = enricher.result(domain)
-          result.value_or nil
+          enricher.result(domain).value_or nil
         end
       end
     end

data/lib/mihari/rule.rb

@@ -2,7 +2,7 @@
 
 module Mihari
   class Rule < Service
-    include Mixins::FalsePositive
+    include Concerns::FalsePositiveValidatable
 
     # @return [Hash]
     attr_reader :data
@@ -84,10 +84,19 @@ module Mihari
     end
 
     #
-    # @return [Array<String>]
+    # @return [Array<Mihari::Models::Tag>]
     #
     def tags
-      data[:tags]
+      data[:tags].uniq.filter_map do |name|
+        Models::Tag.find_or_create_by(name: name)
+      end
+    end
+
+    #
+    # @return [Array<Mihari::Models::Tagging>]
+    #
+    def taggings
+      tags.map { |tag| Models::Tagging.find_or_create_by(tag_id: tag.id, rule_id: id) }
     end
 
     #
@@ -100,8 +109,8 @@ module Mihari
     #
     # @return [Integer, nil]
     #
-    def artifact_lifetime
-      data[:artifact_lifetime] || data[:artifact_ttl]
+    def artifact_ttl
+      data[:artifact_ttl]
     end
 
     #
@@ -111,15 +120,10 @@ module Mihari
     #
     def artifacts
       analyzer_results.flat_map do |result|
-        case result
-        when Success
-          artifacts = result.value!
-          artifacts.map do |artifact|
-            artifact.rule_id = id
-            artifact
-          end
-        else
-          raise result.failure unless analyzer.ignore_error?
+        artifacts = result.value!
+        artifacts.map do |artifact|
+          artifact.rule_id = id
+          artifact
         end
       end
     end
@@ -146,7 +150,7 @@ module Mihari
     #
     def unique_artifacts
       normalized_artifacts.select do |artifact|
-        artifact.unique?(base_time: base_time, artifact_lifetime: artifact_lifetime)
+        artifact.unique?(base_time: base_time, artifact_ttl: artifact_ttl)
       end
     end
 
@@ -206,19 +210,19 @@ module Mihari
     # @return [Mihari::Models::Rule]
     #
     def model
-      rule = Mihari::Models::Rule.find(id)
-
-      rule.title = title
-      rule.description = description
-      rule.data = data
-
-      rule
+      Mihari::Models::Rule.find(id).tap do |rule|
+        rule.title = title
+        rule.description = description
+        rule.data = data
+        rule.taggings = taggings
+      end
     rescue ActiveRecord::RecordNotFound
       Mihari::Models::Rule.new(
         id: id,
         title: title,
         description: description,
-        data: data
+        data: data,
+        taggings: taggings
       )
     end
 
@@ -232,6 +236,13 @@ module Mihari
       false
     end
 
+    #
+    # @return [Boolean]
+    #
+    def exists?
+      Mihari::Models::Rule.exists? id
+    end
+
     def update_or_create
       model.save
     end
@@ -309,7 +320,7 @@ module Mihari
 
     # @return [Array<Dry::Monads::Result::Success<Array<Mihari::Models::Artifact>>, Dry::Monads::Result::Failure>]
     def analyzer_results
-      parallel_results = Parallel.map(parallel_analyzers) { |analyzer| analyzer.result }
+      parallel_results = Parallel.map(parallel_analyzers, &:result)
       serial_results = serial_analyzers.map(&:result)
       parallel_results + serial_results
     end