mihari 7.1.3 → 7.2.0

data/lib/mihari/models/artifact.rb

@@ -28,6 +28,8 @@ module Mihari
       has_many :dns_records, dependent: :destroy
       has_many :ports, dependent: :destroy
       has_many :reverse_dns_names, dependent: :destroy
+      has_many :vulnerabilities, dependent: :destroy
+
       has_many :tags, through: :alert
 
       include ActiveModel::Validations
@@ -38,13 +40,14 @@ module Mihari
         attributes :id, :data, :data_type, :source, :query, :created_at, "alert.id", "rule.id", "rule.title",
           "rule.description"
         attributes tag: "tags.name"
-        attributes asn: "autonomous_system.asn"
+        attributes asn: "autonomous_system.number"
         attributes country_code: "geolocation.country_code"
         attributes "dns_record.value": "dns_records.value"
         attributes "dns_record.resource": "dns_records.resource"
         attributes reverse_dns_name: "reverse_dns_names.name"
         attributes cpe: "cpes.name"
-        attributes port: "ports.port"
+        attributes vuln: "vulnerabilities.name"
+        attributes port: "ports.number"
       end
 
       validates_with ArtifactValidator
@@ -54,6 +57,14 @@ module Mihari
       # @return [String, nil]
       attr_accessor :rule_id
 
+      before_destroy do
+        @alert = alert
+      end
+
+      after_destroy do
+        @alert.destroy unless @alert.artifacts.any?
+      end
+
       #
       # Check uniqueness
       #
@@ -156,6 +167,17 @@ module Mihari
         self.cpes = Services::CPEBuilder.call(data, enricher: enricher)
       end
 
+      #
+      # Enrich vulnerabilities
+      #
+      # @param [Mihari::Enrichers::Shodan] enricher
+      #
+      def enrich_vulnerabilities(enricher = Enrichers::Shodan.new)
+        return unless can_enrich_vulnerabilities?
+
+        self.vulnerabilities = Services::VulnerabilityBuilder.call(data, enricher: enricher)
+      end
+
       #
       # Enrich all the enrichable relationships of the artifact
       #
@@ -167,6 +189,7 @@ module Mihari
         enrich_whois
         enrich_ports shodan
         enrich_cpes shodan
+        enrich_vulnerabilities shodan
       end
 
       ENRICH_METHODS_BY_ENRICHER = {
@@ -181,6 +204,7 @@ module Mihari
           enrich_ports
           enrich_cpes
           enrich_reverse_dns
+          enrich_vulnerabilities
         ],
         Enrichers::GooglePublicDNS => %i[
           enrich_dns
@@ -197,6 +221,45 @@ module Mihari
         methods.each { |method| send(method, enricher) if respond_to?(method) }
       end
 
+      def can_enrich_whois?
+        %w[domain url].include?(data_type) && whois_record.nil?
+      end
+
+      def can_enrich_dns?
+        %w[domain url].include?(data_type) && dns_records.empty?
+      end
+
+      def can_enrich_reverse_dns?
+        data_type == "ip" && reverse_dns_names.empty?
+      end
+
+      def can_enrich_geolocation?
+        data_type == "ip" && geolocation.nil?
+      end
+
+      def can_enrich_autonomous_system?
+        data_type == "ip" && autonomous_system.nil?
+      end
+
+      def can_enrich_ports?
+        data_type == "ip" && ports.empty?
+      end
+
+      def can_enrich_cpes?
+        data_type == "ip" && cpes.empty?
+      end
+
+      def can_enrich_vulnerabilities?
+        data_type == "ip" && vulnerabilities.empty?
+      end
+
+      def enrichable?
+        enrich_methods = methods.map(&:to_s).select { |method| method.start_with?("can_enrich_") }
+        enrich_methods.map(&:to_sym).any? do |method|
+          send(method) if respond_to?(method)
+        end
+      end
+
       class << self
         # @!method search_by_filter(filter)
         #   @param [Mihari::Structs::Filters::Search] filter
@@ -236,34 +299,6 @@ module Mihari
           Addressable::URI.parse(data).host
         end
       end
-
-      def can_enrich_whois?
-        %w[domain url].include?(data_type) && whois_record.nil?
-      end
-
-      def can_enrich_dns?
-        %w[domain url].include?(data_type) && dns_records.empty?
-      end
-
-      def can_enrich_reverse_dns?
-        data_type == "ip" && reverse_dns_names.empty?
-      end
-
-      def can_enrich_geolocation?
-        data_type == "ip" && geolocation.nil?
-      end
-
-      def can_enrich_autonomous_system?
-        data_type == "ip" && autonomous_system.nil?
-      end
-
-      def can_enrich_ports?
-        data_type == "ip" && ports.empty?
-      end
-
-      def can_enrich_cpes?
-        data_type == "ip" && cpes.empty?
-      end
     end
   end
 end

data/lib/mihari/models/vulnerability.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Models
+    #
+    # Vulnerability model
+    #
+    class Vulnerability < ActiveRecord::Base
+      belongs_to :artifact
+    end
+  end
+end

data/lib/mihari/rule.rb

@@ -2,6 +2,7 @@
 
 module Mihari
   class Rule < Service
+    include Concerns::FalsePositiveNormalizable
     include Concerns::FalsePositiveValidatable
 
     # @return [Hash]
@@ -136,8 +137,7 @@ module Mihari
       analyzer_results.flat_map do |result|
         artifacts = result.value!
         artifacts.map do |artifact|
-          artifact.rule_id = id
-          artifact
+          artifact.tap { |tapped| tapped.rule_id = id }
         end
       end
     end
@@ -188,9 +188,7 @@ module Mihari
     def bulk_emit
       return [] if enriched_artifacts.empty?
 
-      Parallel.map(emitters) do |emitter|
-        emitter.result(enriched_artifacts).value_or nil
-      end.compact
+      Parallel.map(emitters) { |emitter| emitter.result(enriched_artifacts).value_or nil }.compact
     end
 
     #
@@ -315,12 +313,12 @@ module Mihari
     # @return [Array<Mihari::Analyzers::Base>]
     #
     def analyzers
-      @analyzers ||= queries.map do |params|
-        analyzer_name = params[:analyzer]
-        klass = get_analyzer_class(analyzer_name)
-        analyzer = klass.from_query(params)
-        analyzer.validate_configuration!
-        analyzer
+      @analyzers ||= queries.deep_dup.map do |params|
+        name = params.delete(:analyzer)
+        klass = get_analyzer_class(name)
+        klass.from_params(params).tap do |analyzer|
+          analyzer.validate_configuration!
+        end
       end
     end
 
@@ -356,16 +354,14 @@ module Mihari
     # @return [Array<Mihari::Emitters::Base>]
     #
     def emitters
-      @emitters ||= data[:emitters].map(&:deep_dup).map do |params|
-        name = params[:emitter]
-        options = params[:options]
-
-        %i[emitter options].each { |key| params.delete key }
+      @emitters ||= data[:emitters].deep_dup.map do |params|
+        name = params.delete(:emitter)
+        options = params.delete(:options)
 
         klass = get_emitter_class(name)
-        emitter = klass.new(rule: self, options: options, **params)
-        emitter.validate_configuration!
-        emitter
+        klass.new(rule: self, options: options, **params).tap do |emitter|
+          emitter.validate_configuration!
+        end
       end
     end
 
@@ -386,11 +382,9 @@ module Mihari
     # @return [Array<Mihari::Enrichers::Base>] enrichers
     #
     def enrichers
-      @enrichers ||= data[:enrichers].map(&:deep_dup).map do |params|
-        name = params[:enricher]
-        options = params[:options]
-
-        %i[enricher options].each { |key| params.delete key }
+      @enrichers ||= data[:enrichers].deep_dup.map do |params|
+        name = params.delete(:enricher)
+        options = params.delete(:options)
 
         klass = get_enricher_class(name)
         klass.new(options: options, **params)

data/lib/mihari/schemas/rule.rb

@@ -45,6 +45,13 @@ module Mihari
           key.failure("#{v} is not a valid format.") unless valid_falsepositive?(v)
         end
       end
+
+      rule(:emitters) do
+        emitters = value.filter_map { |v| v[:emitter] }
+        unless emitters.include?(Mihari::Emitters::Database.key)
+          key.failure("Emitter:#{Mihari::Emitters::Database.key} should be included in emitters")
+        end
+      end
     end
   end
 end

data/lib/mihari/services/builders.rb

@@ -33,7 +33,7 @@ module Mihari
       #
       def call(ip, enricher: Enrichers::MMDB.new)
         enricher.result(ip).fmap do |res|
-          Models::AutonomousSystem.new(asn: res.asn) if res.asn
+          Models::AutonomousSystem.new(number: res.asn) if res.asn
         end.value_or nil
       end
     end
@@ -52,7 +52,7 @@ module Mihari
       #
       def call(ip, enricher: Enrichers::Shodan.new)
         enricher.result(ip).fmap do |res|
-          (res&.cpes || []).map { |cpe| Models::CPE.new(cpe: cpe) }
+          (res&.cpes || []).map { |cpe| Models::CPE.new(name: cpe) }
         end.value_or []
       end
     end
@@ -114,7 +114,7 @@ module Mihari
       #
       def call(ip, enricher: Enrichers::Shodan.new)
         enricher.result(ip).fmap do |res|
-          (res&.ports || []).map { |port| Models::Port.new(port: port) }
+          (res&.ports || []).map { |port| Models::Port.new(number: port) }
         end.value_or []
       end
     end
@@ -138,6 +138,25 @@ module Mihari
       end
     end
 
+    #
+    # Vulnerability builder
+    #
+    class VulnerabilityBuilder < Service
+      #
+      # Build vulnerabilities
+      #
+      # @param [String] ip
+      # @param [Mihari::Enrichers::Shodan] enricher
+      #
+      # @return [Array<Mihari::Models::Vulnerability>]
+      #
+      def call(ip, enricher: Enrichers::Shodan.new)
+        enricher.result(ip).fmap do |res|
+          (res&.vulns || []).map { |name| Models::Vulnerability.new(name: name) }
+        end.value_or []
+      end
+    end
+
     #
     # Whois record builder
     #

data/lib/mihari/services/enrichers.rb

@@ -17,6 +17,8 @@ module Mihari
           :ports
         ).find(id)
 
+        raise UnenrichableError.new, "#{artifact.id} is already enriched or unenrichable" unless artifact.enrichable?
+
         artifact.enrich_all
         artifact.save
       end

data/lib/mihari/services/feed.rb

@@ -93,12 +93,9 @@ module Mihari
       #
       # @param [Object] read_data
       def call(input_enumerator, selector)
-        parsed = proc do
-          $SAFE = 1
-          input_enumerator.instance_eval(selector)
-        end.call
+        parsed = input_enumerator.instance_eval(selector)
 
-        raise TypeError unless parsed.all?(String)
+        raise TypeError unless parsed.is_a?(Array) || parsed.all?(String)
 
         parsed
       end

data/lib/mihari/services/proxies.rb

@@ -79,10 +79,10 @@ module Mihari
       # @return [Mihari::Rule]
       #
       def rule
-        @rule ||= [].tap do |out|
+        @rule ||= lambda do
           data = Mihari::Models::Rule.find(rule_id).data
-          out << Rule.new(**data)
-        end.first
+          Rule.new(**data)
+        end.call
       end
     end
   end

data/lib/mihari/structs/censys.rb

@@ -14,7 +14,7 @@ module Mihari
         # @return [Mihari::AutonomousSystem]
         #
         def as
-          Mihari::Models::AutonomousSystem.new(asn: normalize_asn(asn))
+          Mihari::Models::AutonomousSystem.new(number: normalize_asn(asn))
         end
 
         class << self
@@ -76,7 +76,7 @@ module Mihari
         # @return [Mihari::Port]
         #
         def _port
-          Models::Port.new(port: port)
+          Models::Port.new(number: port)
         end
 
         class << self

data/lib/mihari/structs/greynoise.rb

@@ -22,7 +22,7 @@ module Mihari
         # @return [Mihari::AutonomousSystem]
         #
         def as
-          Mihari::Models::AutonomousSystem.new(asn: normalize_asn(asn))
+          Mihari::Models::AutonomousSystem.new(number: normalize_asn(asn))
         end
 
         #

data/lib/mihari/structs/onyphe.rb

@@ -51,7 +51,7 @@ module Mihari
         # @return [Mihari::AutonomousSystem]
         #
         def as
-          Mihari::Models::AutonomousSystem.new(asn: normalize_asn(asn))
+          Mihari::Models::AutonomousSystem.new(number: normalize_asn(asn))
         end
 
         class << self

data/lib/mihari/structs/shodan.rb

@@ -67,17 +67,25 @@ module Mihari
         #   @return [Integer]
         attribute :port, Types::Int
 
+        # @!attribute [r] cpe
+        #   @return [Array<String>]
+        attribute :cpe, Types::Array(Types::String)
+
+        # @!attribute [r] vulns
+        #   @return [Hash]
+        attribute :vulns, Types::Hash
+
         # @!attribute [r] metadata
         #   @return [Hash]
         attribute :metadata, Types::Hash
 
         #
-        # @return [Mihari::AutonomousSystem, nil]
+        # @return [Mihari::Models::AutonomousSystem, nil]
         #
-        def _asn
+        def autonomous_system
           return nil if asn.nil?
 
-          Mihari::Models::AutonomousSystem.new(asn: normalize_asn(asn))
+          Models::AutonomousSystem.new(number: normalize_asn(asn))
         end
 
         class << self
@@ -103,6 +111,8 @@ module Mihari
               domains: d.fetch("domains"),
               ip_str: d.fetch("ip_str"),
               port: d.fetch("port"),
+              cpe: d["cpe"] || [],
+              vulns: d["vulns"] || {},
               metadata: d
             )
           end
@@ -110,6 +120,8 @@ module Mihari
       end
 
       class Response < Dry::Struct
+        prepend MemoWise
+
         # @!attribute [r] matches
         #   @return [Array<Match>]
         attribute :matches, Types.Array(Match)
@@ -118,6 +130,16 @@ module Mihari
         #   @return [Integer]
         attribute :total, Types::Int
 
+        #
+        # @param [String] ip
+        #
+        # @return [Array<Mihari::Structs::Shodan::Match>]
+        #
+        def select_matches_by_ip(ip)
+          matches.select { |match| match.ip_str == ip }
+        end
+        memo_wise :select_matches_by_ip
+
         #
         # Collect metadata from matches
         #
@@ -126,7 +148,7 @@ module Mihari
         # @return [Array<Hash>]
         #
         def collect_metadata_by_ip(ip)
-          matches.select { |match| match.ip_str == ip }.map(&:metadata)
+          select_matches_by_ip(ip).map(&:metadata)
         end
 
         #
@@ -137,7 +159,7 @@ module Mihari
         # @return [Array<String>]
         #
         def collect_ports_by_ip(ip)
-          matches.select { |match| match.ip_str == ip }.map(&:port)
+          select_matches_by_ip(ip).map(&:port)
         end
 
         #
@@ -148,7 +170,30 @@ module Mihari
         # @return [Array<String>]
         #
         def collect_hostnames_by_ip(ip)
-          matches.select { |match| match.ip_str == ip }.map(&:hostnames).flatten.uniq
+          select_matches_by_ip(ip).map(&:hostnames).flatten.uniq
+        end
+
+        #
+        # Collect CPE from matches
+        #
+        # @param [String] ip
+        #
+        # @return [Array<String>]
+        #
+        def collect_cpes_by_ip(ip)
+          select_matches_by_ip(ip).map(&:cpe).flatten.uniq
+        end
+
+        #
+        # Collect vulnerabilities from matches
+        #
+        # @param [String] ip
+        #
+        # @return [Array<String>]
+        #
+        def collect_vulns_by_ip(ip)
+          # NOTE: vuln keys = CVE IDs
+          select_matches_by_ip(ip).map { |match| match.vulns.keys }.flatten.uniq
         end
 
         #
@@ -158,20 +203,22 @@ module Mihari
           matches.map do |match|
             metadata = collect_metadata_by_ip(match.ip_str)
 
-            ports = collect_ports_by_ip(match.ip_str).map do |port|
-              Mihari::Models::Port.new(port: port)
-            end
+            ports = collect_ports_by_ip(match.ip_str).map { |port| Models::Port.new(number: port) }
             reverse_dns_names = collect_hostnames_by_ip(match.ip_str).map do |name|
-              Mihari::Models::ReverseDnsName.new(name: name)
+              Models::ReverseDnsName.new(name: name)
             end
+            cpes = collect_cpes_by_ip(match.ip_str).map { |name| Models::CPE.new(name: name) }
+            vulnerabilities = collect_vulns_by_ip(match.ip_str).map { |name| Models::Vulnerability.new(name: name) }
 
             Mihari::Models::Artifact.new(
               data: match.ip_str,
               metadata: metadata,
-              autonomous_system: match._asn,
+              autonomous_system: match.autonomous_system,
               geolocation: match.location.geolocation,
               ports: ports,
-              reverse_dns_names: reverse_dns_names
+              reverse_dns_names: reverse_dns_names,
+              cpes: cpes,
+              vulnerabilities: vulnerabilities
             )
           end
         end
@@ -232,15 +279,6 @@ module Mihari
               vulns: d.fetch("vulns")
             )
           end
-
-          #
-          # @param [String] json
-          #
-          # @return [InternetDBResponse]
-          #
-          def from_json!(json)
-            from_dynamic!(JSON.parse(json))
-          end
         end
       end
     end

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "7.1.3"
+  VERSION = "7.2.0"
 end

data/lib/mihari/web/endpoints/artifacts.rb

@@ -53,7 +53,7 @@ module Mihari
 
           desc "Enrich an artifact", {
             success: { code: 201, model: Entities::Message },
-            failure: [{ code: 404, model: Entities::ErrorMessage }],
+            failure: [{ code: 400, model: Entities::ErrorMessage }, { code: 404, model: Entities::ErrorMessage }],
             summary: "Enrich an artifact"
           }
           params do
@@ -74,10 +74,12 @@ module Mihari
               end
             end.to_result
 
-            message = queued ? "ID:#{id}'s enrichment has been queued" : "ID:#{id}'s enrichment has been succeeded"
+            message = queued ? "ID:#{id}'s enrichment is queued" : "ID:#{id}'s enrichment is successful"
             return present({ message: message, queued: queued }, with: Entities::QueueMessage) if result.success?
 
             case result.failure
+            when UnenrichableError
+              error!({ message: result.failure.message }, 400)
             when ActiveRecord::RecordNotFound
               error!({ message: "ID:#{id} not found" }, 404)
             end

data/lib/mihari/web/endpoints/rules.rb

@@ -94,7 +94,7 @@ module Mihari
               end
             end.to_result
 
-            message = queued ? "ID:#{id}'s search has been queued" : "ID:#{id}'s search has been succeed"
+            message = queued ? "ID:#{id}'s search is queued" : "ID:#{id}'s search is successful"
             return present({ message: message, queued: queued }, with: Entities::QueueMessage) if result.success?
 
             case result.failure