mihari 6.1.0 → 6.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dc2d39ad14f53cf88ed2c68bb2e2668e9a43f36444099401ef9fbe1347948644
-  data.tar.gz: 2d08fed6ec6f82da36f72a2fb05ae1ede82ba65d6bc9bcdb7c60b8f7e244c71a
+  metadata.gz: f027c5c24759f4e09d5dad277df4cad58a1705478eba1f3b9b6b354a896a29a5
+  data.tar.gz: a908c432c313bab4f4af70b0b32fb1f2e65c9f40926dc3099c4869330080005c
 SHA512:
-  metadata.gz: 684c4106554f9c11bc7ff7f8633b32c5fa24b4127001415e6d14bd4de73e51a04084eb802a29cac109d8b56c71bc7068c8657f4d6ec7cd53e0519ef353aa859f
-  data.tar.gz: 4a4c3183ca85d47b54f7a98837059a9fae585c2d079fe005336520c62290bc317aa61880695f923a2ad0ad7ba5cf18e3e888990a9394d82b4ac41939f34b3403
+  metadata.gz: 8110df87f854b6e06ea8477c94c6563b2450dae4585738f256da9c94203d14c6fb3ce11e775a470dae04d4d3934549ae71068e6b3f3d3eca48ea3689d87a4cd4
+  data.tar.gz: 375855d7dba59d13ff894472f5f6d5ccc82f36de73aeb12c4144c05863c64283f2ff451dd4531643ba3ddbb218181937d57da1a42bb28970f78170e3fae5dfbf

data/lib/mihari/actor.rb

@@ -65,11 +65,9 @@ module Mihari
 
     def result(...)
       Try[StandardError] do
-        retry_on_error(
-          times: retry_times,
-          interval: retry_interval,
-          exponential_backoff: retry_exponential_backoff
-        ) { call(...) }
+        retry_on_error(times: retry_times, interval: retry_interval, exponential_backoff: retry_exponential_backoff) do
+          call(...)
+        end
       end.to_result
     end
 

data/lib/mihari/analyzers/base.rb

@@ -37,10 +37,14 @@ module Mihari
       # @return [Boolean]
       #
       def ignore_error?
-        ignore_error = options[:ignore_error]
-        return ignore_error unless ignore_error.nil?
+        options[:ignore_error] || Mihari.config.ignore_error
+      end
 
-        Mihari.config.ignore_error
+      #
+      # @return [Boolean]
+      #
+      def parallel?
+        options[:parallel] || Mihari.config.parallel
       end
 
       # @return [Array<String>, Array<Mihari::Models::Artifact>]

data/lib/mihari/analyzers/circl.rb

@@ -26,7 +26,7 @@ module Mihari
       def initialize(query, options: nil, username: nil, password: nil)
         super(refang(query), options: options)
 
-        @type = TypeChecker.type(query)
+        @type = DataType.type(query)
 
         @username = username || Mihari.config.circl_passive_username
         @password = password || Mihari.config.circl_passive_password

data/lib/mihari/analyzers/dnstwister.rb

@@ -18,7 +18,7 @@ module Mihari
       def initialize(query, options: nil)
         super(refang(query), options: options)
 
-        @type = TypeChecker.type(query)
+        @type = DataType.type(query)
       end
 
       def artifacts

data/lib/mihari/analyzers/otx.rb

@@ -22,7 +22,7 @@ module Mihari
       def initialize(query, options: nil, api_key: nil)
         super(refang(query), options: options)
 
-        @type = TypeChecker.type(query)
+        @type = DataType.type(query)
 
         @api_key = api_key || Mihari.config.otx_api_key
       end

data/lib/mihari/analyzers/passivetotal.rb

@@ -26,7 +26,7 @@ module Mihari
       def initialize(query, options: nil, api_key: nil, username: nil)
         super(refang(query), options: options)
 
-        @type = TypeChecker.type(query)
+        @type = DataType.type(query)
 
         @username = username || Mihari.config.passivetotal_username
         @api_key = api_key || Mihari.config.passivetotal_api_key

data/lib/mihari/analyzers/pulsedive.rb

@@ -22,7 +22,7 @@ module Mihari
       def initialize(query, options: nil, api_key: nil)
         super(refang(query), options: options)
 
-        @type = TypeChecker.type(query)
+        @type = DataType.type(query)
 
         @api_key = api_key || Mihari.config.pulsedive_api_key
       end

data/lib/mihari/analyzers/securitytrails.rb

@@ -25,7 +25,7 @@ module Mihari
       def initialize(query, options: nil, api_key: nil)
         super(refang(query), options: options)
 
-        @type = TypeChecker.type(query)
+        @type = DataType.type(query)
 
         @api_key = api_key || Mihari.config.securitytrails_api_key
       end

data/lib/mihari/analyzers/virustotal.rb

@@ -22,7 +22,7 @@ module Mihari
       def initialize(query, options: nil, api_key: nil)
         super(refang(query), options: options)
 
-        @type = TypeChecker.type(query)
+        @type = DataType.type(query)
 
         @api_key = api_key || Mihari.config.virustotal_api_key
       end

data/lib/mihari/clients/google_public_dns.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Clients
+    #
+    # Google Public DNS enricher
+    #
+    class GooglePublicDNS < Base
+      #
+      # @param [String] base_url
+      # @param [Hash] headers
+      # @param [Integer, nil] timeout
+      #
+      def initialize(base_url = "https://dns.google", headers: {}, timeout: nil)
+        super(base_url, headers: headers, timeout: timeout)
+      end
+
+      #
+      # Query Google Public DNS by resource type
+      #
+      # @param [String] name
+      #
+      # @return [Mihari::Structs::GooglePublicDNS::Response, nil]
+      #
+      def query_all(name)
+        Structs::GooglePublicDNS::Response.from_dynamic! get_json("/resolve",
+          params: { name: name, type: "ALL" })
+      end
+    end
+  end
+end

data/lib/mihari/config.rb

@@ -42,6 +42,7 @@ module Mihari
       ignore_error: false,
       pagination_interval: 0,
       pagination_limit: 100,
+      parallel: false,
       retry_exponential_backoff: true,
       retry_interval: 5,
       retry_times: 3,
@@ -62,7 +63,7 @@ module Mihari
     #   @return [String, nil]
 
     # @!attribute [r] database_url
-    #   @return [String, nil]
+    #   @return [URI, nil]
 
     # @!attribute [r] fofa_api_key
     #   @return [String, nil]
@@ -151,6 +152,9 @@ module Mihari
     # @!attribute [r] pagination_limit
     #   @return [Integer]
 
+    # @!attribute [r] parallel
+    #   @return [Boolean]
+
     # @!attribute [r] ignore_error
     #   @return [Boolean]
 

data/lib/mihari/{type_checker.rb → data_type.rb}

@@ -2,9 +2,11 @@
 
 module Mihari
   #
-  # Artifact type checker
+  # (Artifact) Data Type
   #
-  class TypeChecker
+  class DataType
+    include Dry::Monads[:try]
+
     # @return [String]
     attr_reader :data
 
@@ -24,26 +26,25 @@ module Mihari
 
     # @return [Boolean]
     def ip?
-      IPAddr.new data
-      true
-    rescue IPAddr::InvalidAddressError => _e
-      false
+      Try[IPAddr::InvalidAddressError] do
+        IPAddr.new(data).to_s == data
+      end.to_result.value_or(false)
     end
 
     # @return [Boolean]
     def domain?
-      uri = Addressable::URI.parse("http://#{data}")
-      uri.host == data && PublicSuffix.valid?(uri.host)
-    rescue Addressable::URI::InvalidURIError => _e
-      false
+      Try[Addressable::URI::InvalidURIError] do
+        uri = Addressable::URI.parse("http://#{data}")
+        uri.host == data && PublicSuffix.valid?(uri.host)
+      end.to_result.value_or(false)
     end
 
     # @return [Boolean]
     def url?
-      uri = Addressable::URI.parse(data)
-      uri.scheme && uri.host && uri.path && PublicSuffix.valid?(uri.host)
-    rescue Addressable::URI::InvalidURIError => _e
-      false
+      Try[Addressable::URI::InvalidURIError] do
+        uri = Addressable::URI.parse(data)
+        uri.scheme && uri.host && uri.path && PublicSuffix.valid?(uri.host)
+      end.to_result.value_or(false)
     end
 
     # @return [Boolean]
@@ -53,38 +54,20 @@ module Mihari
 
     # @return [String, nil]
     def type
-      return "hash" if hash?
-      return "ip" if ip?
-      return "domain" if domain?
-      return "url" if url?
+      found = %i[hash? ip? domain? url? mail?].find { |method| send(method) if respond_to?(method) }
+      return nil if found.nil?
 
-      "mail" if mail?
+      found[...-1].to_s
     end
 
     # @return [String, nil]
     def detailed_type
-      return "md5" if md5?
-      return "sha1" if sha1?
-      return "sha256" if sha256?
-      return "sha512" if sha512?
+      found = %i[md5? sha1? sha256? sha512?].find { |method| send(method) if respond_to?(method) }
+      return found[...-1].to_s unless found.nil?
 
       type
     end
 
-    class << self
-      # @return [String, nil]
-      def type(data)
-        new(data).type
-      end
-
-      # @return [String, nil]
-      def detailed_type(data)
-        new(data).detailed_type
-      end
-    end
-
-    private
-
     # @return [Boolean]
     def md5?
       data.match?(/^[A-Fa-f0-9]{32}$/)
@@ -104,5 +87,17 @@ module Mihari
     def sha512?
       data.match?(/^[A-Fa-f0-9]{128}$/)
     end
+
+    class << self
+      # @return [String, nil]
+      def type(data)
+        new(data).type
+      end
+
+      # @return [String, nil]
+      def detailed_type(data)
+        new(data).detailed_type
+      end
+    end
   end
 end

data/lib/mihari/database.rb

@@ -154,7 +154,7 @@ module Mihari
 
         case adapter
         when "postgresql", "mysql2"
-          ActiveRecord::Base.establish_connection(Mihari.config.database_url.to_s)
+          ActiveRecord::Base.establish_connection Mihari.config.database_url.to_s
         else
           ActiveRecord::Base.establish_connection(
             adapter: adapter,
@@ -162,8 +162,6 @@ module Mihari
           )
         end
         ActiveRecord::Base.logger = Logger.new($stdout) if development_env?
-      rescue StandardError => e
-        Mihari.logger.error e
       end
 
       #

data/lib/mihari/enrichers/google_public_dns.rb

@@ -11,27 +11,10 @@ module Mihari
       #
       # @param [String] name
       #
-      # @return [Array<Mihari::Structs::GooglePublicDNS::Response>]
+      # @return [Mihari::Structs::GooglePublicDNS::Response]
       #
       def call(name)
-        %w[A AAAA CNAME TXT NS].filter_map { |resource_type| query_by_type(name, resource_type) }
-      end
-
-      #
-      # Query Google Public DNS by resource type
-      #
-      # @param [String] name
-      # @param [String] resource_type
-      #
-      # @return [Mihari::Structs::GooglePublicDNS::Response, nil]
-      #
-      def query_by_type(name, resource_type)
-        url = "https://dns.google/resolve"
-        params = { name: name, type: resource_type }
-        res = http.get(url, params: params)
-        Structs::GooglePublicDNS::Response.from_dynamic! JSON.parse(res.body.to_s)
-      rescue HTTPError
-        nil
+        client.query_all name
       end
 
       class << self
@@ -45,8 +28,8 @@ module Mihari
 
       private
 
-      def http
-        HTTP::Factory.build timeout: timeout
+      def client
+        Clients::GooglePublicDNS.new
       end
     end
   end

data/lib/mihari/entities/artifact.rb

@@ -42,5 +42,13 @@ module Mihari
         status.ports.empty? ? nil : status.ports
       end
     end
+
+    class ArtifactsWithPagination < Grape::Entity
+      expose :artifacts, using: Entities::Artifact,
+        documentation: { type: Entities::Artifact, is_array: true, required: true }
+      expose :total, documentation: { type: Integer, required: true }
+      expose :current_page, documentation: { type: Integer, required: true }, as: :currentPage
+      expose :page_size, documentation: { type: Integer, required: true }, as: :pageSize
+    end
   end
 end

data/lib/mihari/models/alert.rb

@@ -30,8 +30,7 @@ module Mihari
           offset = (page - 1) * limit
 
           relation = build_relation(filter.without_pagination)
-          alert_ids = relation.limit(limit).offset(offset).order(id: :desc).pluck(:id).uniq
-          eager_load(:artifacts, :tags).where(id: [alert_ids]).order(id: :desc)
+          relation.limit(limit).offset(offset).order(id: :desc)
         end
 
         #
@@ -48,39 +47,17 @@ module Mihari
 
         private
 
-        #
-        # @param [Mihari::Structs::Filters::Alert::SearchFilter] filter
-        #
-        # @return [Array<Integer>]
-        #
-        def get_artifact_ids_by_filter(filter)
-          artifact_ids = []
-
-          if filter.artifact_data
-            artifact = Artifact.where(data: filter.artifact_data)
-            artifact_ids = artifact.pluck(:id)
-            # set invalid ID if nothing is matched with the filters
-            artifact_ids = [-1] if artifact_ids.empty?
-          end
-
-          artifact_ids
-        end
-
         #
         # @param [Mihari::Structs::Filters::Alert::SearchFilter] filter
         #
         # @return [Mihari::Models::Alert]
         #
         def build_relation(filter)
-          artifact_ids = get_artifact_ids_by_filter(filter)
-
-          relation = includes(:artifacts, :tags)
-
-          relation = relation.where(artifacts: { id: artifact_ids }) unless artifact_ids.empty?
-          relation = relation.where(tags: { name: filter.tag_name }) if filter.tag_name
+          relation = eager_load(:artifacts, :tags)
 
+          relation = relation.where(artifacts: { data: filter.artifact }) if filter.artifact
+          relation = relation.where(tags: { name: filter.tag }) if filter.tag
           relation = relation.where(rule_id: filter.rule_id) if filter.rule_id
-
           relation = relation.where("alerts.created_at >= ?", filter.from_at) if filter.from_at
           relation = relation.where("alerts.created_at <= ?", filter.to_at) if filter.to_at
 

data/lib/mihari/models/artifact.rb

@@ -46,7 +46,7 @@ module Mihari
 
         super(*args, **kwargs)
 
-        self.data_type = TypeChecker.type(data)
+        self.data_type = DataType.type(data)
 
         @tags = []
         @rule_id = ""
@@ -77,6 +77,18 @@ module Mihari
         artifact.created_at < decayed_at
       end
 
+      #
+      # Count artifacts
+      #
+      # @param [Mihari::Structs::Filters::Artifact::SearchFilter] filter
+      #
+      # @return [Integer]
+      #
+      def count(filter)
+        relation = build_relation(filter)
+        relation.distinct("artifact.id").count
+      end
+
       #
       # Enrich whois record
       #
@@ -105,7 +117,7 @@ module Mihari
       # @param [Mihari::Enrichers::Shodan] enricher
       #
       def enrich_reverse_dns(enricher = Enrichers::Shodan.new)
-        return unless can_enrich_revese_dns?
+        return unless can_enrich_reverse_dns?
 
         self.reverse_dns_names = ReverseDnsName.build_by_ip(data, enricher: enricher)
       end
@@ -195,6 +207,56 @@ module Mihari
         methods.each { |method| send(method, enricher) if respond_to?(method) }
       end
 
+      class << self
+        #
+        # Search artifacts
+        #
+        # @param [Mihari::Structs::Filters::Artifact::SearchFilterWithPagination] filter
+        #
+        # @return [Array<Artifact>]
+        #
+        def search(filter)
+          limit = filter.limit.to_i
+          raise ArgumentError, "limit should be bigger than zero" unless limit.positive?
+
+          page = filter.page.to_i
+          raise ArgumentError, "page should be bigger than zero" unless page.positive?
+
+          offset = (page - 1) * limit
+
+          relation = build_relation(filter.without_pagination)
+          relation.limit(limit).offset(offset).order(id: :desc)
+        end
+
+        #
+        # Count artifacts
+        #
+        # @param [Mihari::Structs::Filters::Artifact::SearchFilter] filter
+        #
+        # @return [Integer]
+        #
+        def count(filter)
+          relation = build_relation(filter)
+          relation.distinct("artifacts.id").count
+        end
+
+        #
+        # @param [Mihari::Structs::Filters::Artifact::SearchFilter] filter
+        #
+        # @return [Mihari::Models::Artifact]
+        #
+        def build_relation(filter)
+          relation = eager_load(alert: :tags)
+
+          relation = relation.where(alert: { rule_id: filter.rule_id }) if filter.rule_id
+          relation = relation.where(alert: { tags: { name: filter.tag } }) if filter.tag
+          relation = relation.where("artifacts.created_at >= ?", filter.from_at) if filter.from_at
+          relation = relation.where("artifacts.created_at <= ?", filter.to_at) if filter.to_at
+
+          relation
+        end
+      end
+
       private
 
       def ipinfo
@@ -219,7 +281,7 @@ module Mihari
         %w[domain url].include?(data_type) && dns_records.empty?
       end
 
-      def can_enrich_revese_dns?
+      def can_enrich_reverse_dns?
         data_type == "ip" && reverse_dns_names.empty?
       end
 

data/lib/mihari/models/dns.rb

@@ -20,16 +20,11 @@ module Mihari
         # @return [Array<Mihari::Models::DnsRecord>]
         #
         def build_by_domain(domain, enricher: Enrichers::GooglePublicDNS.new)
-          result = enricher.result(domain).bind do |responses|
+          enricher.result(domain).bind do |res|
             Success(
-              responses.map do |res|
-                res.answers.map do |answer|
-                  new(resource: answer.resource_type, value: answer.data)
-                end
-              end.flatten
+              res.answers.map { |answer| new(resource: answer.resource_type, value: answer.data) }
             )
-          end
-          result.value_or []
+          end.value_or([])
         end
       end
     end

data/lib/mihari/models/rule.rb

@@ -40,10 +40,7 @@ module Mihari
           offset = (page - 1) * limit
 
           relation = build_relation(filter.without_pagination)
-
-          # TODO: improve queires
-          rule_ids = relation.limit(limit).offset(offset).order(created_at: :desc).pluck(:id).uniq
-          where(id: [rule_ids]).order(created_at: :desc)
+          relation.limit(limit).offset(offset).order(created_at: :desc)
         end
 
         #
@@ -68,7 +65,7 @@ module Mihari
         def build_relation(filter)
           relation = includes(alerts: :tags)
 
-          relation = relation.where(alerts: { tags: { name: filter.tag_name } }) if filter.tag_name
+          relation = relation.where(alerts: { tags: { name: filter.tag } }) if filter.tag
 
           relation = relation.where("rules.title LIKE ?", "%#{filter.title}%") if filter.title
           relation = relation.where("rules.description LIKE ?", "%#{filter.description}%") if filter.description

data/lib/mihari/rule.rb

@@ -110,9 +110,7 @@ module Mihari
     # @return [Array<Mihari::Models::Artifact>]
     #
     def artifacts
-      analyzers.flat_map do |analyzer|
-        # @type [Dry::Monads::Result::Success<Array<Mihari::Models::Artifact>>, Dry::Monads::Result::Failure]
-        result = analyzer.result
+      analyzer_results.flat_map do |result|
         case result
         when Success
           artifacts = result.value!
@@ -123,7 +121,7 @@ module Mihari
         else
           raise result.failure unless analyzer.ignore_error?
         end
-      end.compact
+      end
     end
 
     #
@@ -292,15 +290,30 @@ module Mihari
     # @return [Array<Mihari::Analyzers::Base>]
     #
     def analyzers
-      @analyzers ||= queries.map do |query_params|
-        analyzer_name = query_params[:analyzer]
+      @analyzers ||= queries.map do |params|
+        analyzer_name = params[:analyzer]
         klass = get_analyzer_class(analyzer_name)
-        analyzer = klass.from_query(query_params)
+        analyzer = klass.from_query(params)
         analyzer.validate_configuration!
         analyzer
       end
     end
 
+    def parallel_analyzers
+      analyzers.select(&:parallel?)
+    end
+
+    def serial_analyzers
+      analyzers.reject(&:parallel?)
+    end
+
+    # @return [Array<Dry::Monads::Result::Success<Array<Mihari::Models::Artifact>>, Dry::Monads::Result::Failure>]
+    def analyzer_results
+      parallel_results = Parallel.map(parallel_analyzers) { |analyzer| analyzer.result }
+      serial_results = serial_analyzers.map(&:result)
+      parallel_results + serial_results
+    end
+
     #
     # Get emitter class
     #

data/lib/mihari/schemas/options.rb

@@ -15,7 +15,11 @@ module Mihari
       optional(:ignore_error).value(:bool).default(Mihari.config.ignore_error)
     end
 
-    AnalyzerOptions = Options | IgnoreErrorOptions
+    ParallelOptions = Dry::Schema.Params do
+      optional(:parallel).value(:bool).default(Mihari.config.parallel)
+    end
+
+    AnalyzerOptions = Options | IgnoreErrorOptions | ParallelOptions
 
     PaginationOptions = Dry::Schema.Params do
       optional(:pagination_interval).value(:integer).default(Mihari.config.pagination_interval)