mihari 5.7.2 → 6.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5c7fa4de39ec815c08930311d4ed642857236021301508c47d298cba157dd811
-  data.tar.gz: 1598a9992be6739957bc5145c11d93f8a869588d42985799d527703178f78a8f
+  metadata.gz: dc2d39ad14f53cf88ed2c68bb2e2668e9a43f36444099401ef9fbe1347948644
+  data.tar.gz: 2d08fed6ec6f82da36f72a2fb05ae1ede82ba65d6bc9bcdb7c60b8f7e244c71a
 SHA512:
-  metadata.gz: 6c96715d51778d724b4df1c502d3083c4af084997019cc7ef43fc5d131cfeb02c4b0f06d6353c0d8e75e26dec0fb20038fa36718f9a65bb1a3a771224442bc3e
-  data.tar.gz: 655c0efa77ff2602fef509b351ebf02ab9c222728ddfcfc0da36ba8fa27a4e1518f1d70406b5ba3ace86012c196954a98c9af10a4de58c5dcdf73bbc760d2056
+  metadata.gz: 684c4106554f9c11bc7ff7f8633b32c5fa24b4127001415e6d14bd4de73e51a04084eb802a29cac109d8b56c71bc7068c8657f4d6ec7cd53e0519ef353aa859f
+  data.tar.gz: 4a4c3183ca85d47b54f7a98837059a9fae585c2d079fe005336520c62290bc317aa61880695f923a2ad0ad7ba5cf18e3e888990a9394d82b4ac41939f34b3403

data/.rubocop.yml

@@ -13,6 +13,10 @@ Metrics/MethodLength:
   Max: 50
 Metrics/AbcSize:
   Max: 50
+RSpec/MultipleMemoizedHelpers:
+  Max: 10
+RSpec/ExampleLength:
+  Max: 20
 require:
   - rubocop-rspec
   - rubocop-yard

data/config.ru

@@ -1,5 +1,7 @@
 require "./lib/mihari"
 
+require "better_errors"
+
 # set rack env as development
 ENV["RACK_ENV"] ||= "development"
 

data/lib/mihari/actor.rb

@@ -78,7 +78,7 @@ module Mihari
       # @return [String]
       #
       def class_key
-        to_s.split("::").last
+        to_s.split("::").last.downcase
       end
 
       #

data/lib/mihari/analyzers/base.rb

@@ -60,7 +60,10 @@ module Mihari
           # No need to set data_type manually
           # It is set automatically in #initialize
           artifact = artifact.is_a?(Models::Artifact) ? artifact : Models::Artifact.new(data: artifact)
+
           artifact.source = self.class.class_key
+          artifact.query = query
+
           artifact
         end.select(&:valid?).uniq(&:data)
       end

data/lib/mihari/analyzers/dnstwister.rb

@@ -25,9 +25,7 @@ module Mihari
         raise ValueError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
 
         domains = client.fuzz(query)
-        Parallel.map(domains) do |domain|
-          resolvable?(domain) ? domain : nil
-        end.compact
+        Parallel.map(domains) { |domain| resolvable?(domain) ? domain : nil }.compact
       end
 
       private
@@ -55,7 +53,7 @@ module Mihari
       def resolvable?(domain)
         Resolv.getaddress domain
         true
-      rescue Resolv::ResolvError => _e
+      rescue Resolv::ResolvError
         false
       end
     end

data/lib/mihari/analyzers/hunterhow.rb

@@ -22,7 +22,7 @@ module Mihari
       # @param [Hash, nil] options
       # @param [String, nil] api_key
       #
-      def initialize(query, start_time:, end_time:, options: nil, api_key: nil)
+      def initialize(query, start_time: nil, end_time: nil, options: nil, api_key: nil)
         super(query, options: options)
 
         @api_key = api_key || Mihari.config.hunterhow_api_key

data/lib/mihari/analyzers/urlscan.rb

@@ -34,10 +34,7 @@ module Mihari
       def artifacts
         # @type [Array<Mihari::Models::Artifact>]
         artifacts = client.search_with_pagination(query, pagination_limit: pagination_limit).map(&:artifacts).flatten
-
-        artifacts.select do |artifact|
-          allowed_data_types.include? artifact.data_type
-        end
+        artifacts.select { |artifact| allowed_data_types.include? artifact.data_type }
       end
 
       def configuration_keys

data/lib/mihari/cli/main.rb

@@ -32,19 +32,9 @@ module Mihari
       include Mihari::Commands::Version
       include Mihari::Commands::Web
 
-      no_commands do
-        def unwrap_error(err)
-          return err unless err.is_a?(Dry::Monads::UnwrapError)
-
-          # NOTE: UnwrapError's receiver can be either of:
-          #       - Dry::Monads::Try::Error
-          #       - Dry::Monads::Result::Failure
-          receiver = err.receiver
-          return receiver.exception if receiver.is_a?(Dry::Monads::Try::Error)
-
-          receiver.failure
-        end
+      include Mihari::Mixins::UnwrapError
 
+      no_commands do
         def safe_execute
           yield
         rescue StandardError => e

data/lib/mihari/commands/database.rb

@@ -19,7 +19,6 @@ module Mihari
             #
             def migrate(direction = "up")
               ActiveRecord::Migration.verbose = options["verbose"]
-
               Mihari::Database.migrate direction.to_sym
             end
           end

data/lib/mihari/config.rb

@@ -45,7 +45,8 @@ module Mihari
       retry_exponential_backoff: true,
       retry_interval: 5,
       retry_times: 3,
-      sentry_dsn: nil
+      sentry_dsn: nil,
+      sentry_trace_sample_rate: 0.25
     )
 
     # @!attribute [r] binaryedge_api_key
@@ -132,6 +133,9 @@ module Mihari
     # @!attribute [r] sentry_dsn
     #   @return [String, nil]
 
+    # @!attribute [r] sentry_trace_sample_rate
+    #   @return [Float]
+
     # @!attribute [r] retry_interval
     #   @return [Integer]
 

data/lib/mihari/database.rb

@@ -1,9 +1,7 @@
 # frozen_string_literal: true
 
 # Make possible to use upper case acronyms in class names
-ActiveSupport::Inflector.inflections(:en) do |inflect|
-  inflect.acronym "CPE"
-end
+ActiveSupport::Inflector.inflections(:en) { |inflect| inflect.acronym "CPE" }
 
 def env
   ENV["APP_ENV"] || ENV["RACK_ENV"]
@@ -113,6 +111,12 @@ class V5Schema < ActiveRecord::Migration[7.1]
   end
 end
 
+class V61Schema < ActiveRecord::Migration[7.1]
+  def change
+    add_column :artifacts, :query, :string
+  end
+end
+
 def adapter
   return "postgresql" if %w[postgresql postgres].include?(Mihari.config.database_url.scheme)
   return "mysql2" if Mihari.config.database_url.scheme == "mysql2"
@@ -124,7 +128,7 @@ end
 # @return [Array<ActiveRecord::Migration>] schemas
 #
 def schemas
-  [V5Schema]
+  [V5Schema, V61Schema]
 end
 
 module Mihari
@@ -175,7 +179,7 @@ module Mihari
         Mihari::Database.connect
         yield
       rescue ActiveRecord::StatementInvalid
-        Mihari.logger.error("You haven't finished the DB migration! Please run 'mihari db migrate'.")
+        Mihari.logger.error("The DB migration is not yet complete. Please run 'mihari db migrate'.")
       ensure
         Mihari::Database.close
       end

data/lib/mihari/emitters/misp.rb

@@ -56,12 +56,12 @@ module Mihari
         })
       end
 
-      private
-
       def configuration_keys
         %w[misp_url misp_api_key]
       end
 
+      private
+
       def client
         @client ||= Clients::MISP.new(url, api_key: api_key, timeout: timeout)
       end

data/lib/mihari/emitters/slack.rb

@@ -6,7 +6,7 @@ require "slack-notifier"
 module Mihari
   module Emitters
     class Attachment
-      include Memist::Memoizable
+      prepend MemoWise
 
       # @return [String]
       attr_reader :data
@@ -76,7 +76,7 @@ module Mihari
           "https://urlscan.io/domain/#{uri.hostname}"
         end
       end
-      memoize :_urlscan_link
+      memo_wise :_urlscan_link
 
       # @return [String, nil]
       def _vt_link
@@ -93,19 +93,19 @@ module Mihari
           "https://www.virustotal.com/#/search/#{data}"
         end
       end
-      memoize :_vt_link
+      memo_wise :_vt_link
 
       # @return [String, nil]
       def _censys_link
         (data_type == "ip") ? "https://search.censys.io/hosts/#{data}" : nil
       end
-      memoize :_censys_link
+      memo_wise :_censys_link
 
       # @return [String, nil]
       def _shodan_link
         (data_type == "ip") ? "https://www.shodan.io/host/#{data}" : nil
       end
-      memoize :_shodan_link
+      memo_wise :_shodan_link
 
       # @return [String]
       def sha256
@@ -192,9 +192,7 @@ module Mihari
       # @return [Array<Mihari::Emitters::Attachment>]
       #
       def attachments
-        artifacts.map do |artifact|
-          Attachment.new(data: artifact.data, data_type: artifact.data_type).to_a
-        end.flatten
+        artifacts.map { |artifact| Attachment.new(data: artifact.data, data_type: artifact.data_type).to_a }.flatten
       end
 
       #
@@ -205,7 +203,6 @@ module Mihari
       def text
         tags = rule.tags
         tags = ["N/A"] if tags.empty?
-
         [
           "*#{rule.title}*",
           "*Desc.*: #{rule.description}",
@@ -217,10 +214,10 @@ module Mihari
       # @param [Array<Mihari::Models::Artifact>] artifacts
       #
       def call(artifacts)
-        return if artifacts.empty?
-
         @artifacts = artifacts
 
+        return if artifacts.empty?
+
         notifier.post(text: text, attachments: attachments, mrkdwn: true)
       end
 

data/lib/mihari/emitters/the_hive.rb

@@ -43,10 +43,10 @@ module Mihari
       # @param [Array<Mihari::Models::Artifact>] artifacts
       #
       def call(artifacts)
-        return if artifacts.empty?
-
         @artifacts = artifacts
 
+        return if artifacts.empty?
+
         client.alert payload
       end
 
@@ -61,21 +61,17 @@ module Mihari
         @normalized_api_version ||= [].tap do |out|
           # v4 does not have version prefix in path (/api/)
           # v5 has version prefix in path (/api/v1/)
-          table = {
-            "" => nil,
-            "v4" => nil,
-            "v5" => "v1"
-          }
+          table = { "" => nil, "v4" => nil, "v5" => "v1" }
           out << table[api_version.to_s.downcase]
         end.first
       end
 
-      private
-
       def configuration_keys
         %w[thehive_url thehive_api_key]
       end
 
+      private
+
       def client
         @client ||= Clients::TheHive.new(url, api_key: api_key, api_version: normalized_api_version, timeout: timeout)
       end

data/lib/mihari/enrichers/base.rb

@@ -6,6 +6,8 @@ module Mihari
     # Base class for enrichers
     #
     class Base < Actor
+      prepend MemoWise
+
       def initialize(options: nil)
         super(options: options)
       end

data/lib/mihari/enrichers/google_public_dns.rb

@@ -14,9 +14,7 @@ module Mihari
       # @return [Array<Mihari::Structs::GooglePublicDNS::Response>]
       #
       def call(name)
-        %w[A AAAA CNAME TXT NS].filter_map do |resource_type|
-          query_by_type(name, resource_type)
-        end
+        %w[A AAAA CNAME TXT NS].filter_map { |resource_type| query_by_type(name, resource_type) }
       end
 
       #
@@ -31,10 +29,7 @@ module Mihari
         url = "https://dns.google/resolve"
         params = { name: name, type: resource_type }
         res = http.get(url, params: params)
-
-        data = JSON.parse(res.body.to_s)
-
-        Structs::GooglePublicDNS::Response.from_dynamic! data
+        Structs::GooglePublicDNS::Response.from_dynamic! JSON.parse(res.body.to_s)
       rescue HTTPError
         nil
       end

data/lib/mihari/enrichers/ipinfo.rb

@@ -33,10 +33,9 @@ module Mihari
       def call(ip)
         url = "https://ipinfo.io/#{ip}/json"
         res = http.get(url)
-        data = JSON.parse(res.body.to_s)
-
-        Structs::IPInfo::Response.from_dynamic! data
+        Structs::IPInfo::Response.from_dynamic! JSON.parse(res.body.to_s)
       end
+      memo_wise :call
 
       private
 

data/lib/mihari/enrichers/shodan.rb

@@ -16,10 +16,9 @@ module Mihari
       def call(ip)
         url = "https://internetdb.shodan.io/#{ip}"
         res = http.get(url)
-        data = JSON.parse(res.body.to_s)
-
-        Structs::Shodan::InternetDBResponse.from_dynamic! data
+        Structs::Shodan::InternetDBResponse.from_dynamic! JSON.parse(res.body.to_s)
       end
+      memo_wise :call
 
       private
 

data/lib/mihari/enrichers/whois.rb

@@ -8,16 +8,11 @@ module Mihari
     # Whois enricher
     #
     class Whois < Base
-      # @return [Hash]
-      attr_accessor :memo
-
       #
       # @param [Hash, nil] options
       #
       def initialize(options: nil)
         super(options: options)
-
-        @memo = {}
       end
 
       #
@@ -28,16 +23,22 @@ module Mihari
       # @return [Mihari::Models::WhoisRecord, nil]
       #
       def call(domain)
-        domain = PublicSuffix.domain(domain)
+        _call PublicSuffix.domain(domain)
+      end
 
-        # check memo
-        return memo[domain].dup if memo.key?(domain)
+      private
 
+      #
+      # @param [String] domain
+      #
+      # @return [Mihari::Models::WhoisRecord, nil]
+      #
+      def _call(domain)
         record = whois.lookup(domain)
         parser = record.parser
         return nil if parser.available?
 
-        whois_record = Models::WhoisRecord.new(
+        Models::WhoisRecord.new(
           domain: domain,
           created_on: get_created_on(parser),
           updated_on: get_updated_on(parser),
@@ -45,18 +46,8 @@ module Mihari
           registrar: get_registrar(parser),
           contacts: get_contacts(parser)
         )
-
-        # set memo
-        memo[domain] = whois_record
-
-        whois_record
       end
-
-      def reset_cache
-        @memo = {}
-      end
-
-      private
+      memo_wise :_call
 
       #
       # @return [::Whois::Client]

data/lib/mihari/entities/artifact.rb

@@ -11,6 +11,7 @@ module Mihari
       expose :data, documentation: { type: String, required: true }
       expose :data_type, documentation: { type: String, required: true }, as: :dataType
       expose :source, documentation: { type: String, required: true }
+      expose :query, documentation: { type: String, required: false }
       expose :tags, documentation: { type: String, is_array: true }
     end
 

data/lib/mihari/mixins/falsepositive.rb

@@ -6,7 +6,7 @@ module Mihari
     # False positive mixins
     #
     module FalsePositive
-      include Memist::Memoizable
+      prepend MemoWise
 
       #
       # Normalize a falsepositive value
@@ -22,7 +22,7 @@ module Mihari
         value_without_slashes = value[1..-2]
         Regexp.compile value_without_slashes.to_s
       end
-      memoize :normalize_falsepositive
+      memo_wise :normalize_falsepositive
 
       #
       # Check whether a value is valid format as a disallowed data value

data/lib/mihari/mixins/refang.rb

@@ -14,10 +14,7 @@ module Mihari
       # @return [String]
       #
       def refang(indicator)
-        return indicator.gsub("[.]", ".").gsub("(.)", ".") if indicator.is_a?(String)
-
-        # for RSpec & Ruby 2.7
-        indicator
+        indicator.gsub("[.]", ".").gsub("(.)", ".")
       end
     end
   end

data/lib/mihari/mixins/unwrap_error.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Mixins
+    #
+    # Unwrap error mixins
+    #
+    module UnwrapError
+      def unwrap_error(err)
+        return err unless err.is_a?(Dry::Monads::UnwrapError)
+
+        # NOTE: UnwrapError's receiver can be either of:
+        #       - Dry::Monads::Try::Error
+        #       - Dry::Monads::Result::Failure
+        receiver = err.receiver
+        case receiver
+        when Dry::Monads::Try::Error
+          receiver.exception
+        when Dry::Monads::Failure
+          receiver.failure
+        else
+          err
+        end
+      end
+    end
+  end
+end

data/lib/mihari/models/alert.rb

@@ -30,7 +30,6 @@ module Mihari
           offset = (page - 1) * limit
 
           relation = build_relation(filter.without_pagination)
-
           alert_ids = relation.limit(limit).offset(offset).order(id: :desc).pluck(:id).uniq
           eager_load(:artifacts, :tags).where(id: [alert_ids]).order(id: :desc)
         end
@@ -75,8 +74,7 @@ module Mihari
         def build_relation(filter)
           artifact_ids = get_artifact_ids_by_filter(filter)
 
-          relation = self
-          relation = relation.includes(:artifacts, :tags)
+          relation = includes(:artifacts, :tags)
 
           relation = relation.where(artifacts: { id: artifact_ids }) unless artifact_ids.empty?
           relation = relation.where(tags: { name: filter.tag_name }) if filter.tag_name

data/lib/mihari/models/artifact.rb

@@ -78,7 +78,7 @@ module Mihari
       end
 
       #
-      # Enrich(add) whois record
+      # Enrich whois record
       #
       # @param [Mihari::Enrichers::Whois] enricher
       #
@@ -89,7 +89,7 @@ module Mihari
       end
 
       #
-      # Enrich(add) DNS records
+      # Enrich DNS records
       #
       # @param [Mihari::Enrichers::GooglePublicDNS] enricher
       #
@@ -100,7 +100,7 @@ module Mihari
       end
 
       #
-      # Enrich(add) reverse DNS names
+      # Enrich reverse DNS names
       #
       # @param [Mihari::Enrichers::Shodan] enricher
       #
@@ -111,7 +111,7 @@ module Mihari
       end
 
       #
-      # Enrich(add) geolocation
+      # Enrich geolocation
       #
       # @param [Mihari::Enrichers::IPInfo] enricher
       #
@@ -158,13 +158,13 @@ module Mihari
       # Enrich all the enrichable relationships of the artifact
       #
       def enrich_all
-        enrich_autonomous_system
+        enrich_autonomous_system ipinfo
         enrich_dns
-        enrich_geolocation
-        enrich_reverse_dns
+        enrich_geolocation ipinfo
+        enrich_reverse_dns shodan
         enrich_whois
-        enrich_ports
-        enrich_cpes
+        enrich_ports shodan
+        enrich_cpes shodan
       end
 
       ENRICH_METHODS_BY_ENRICHER = {
@@ -192,13 +192,19 @@ module Mihari
       #
       def enrich_by_enricher(enricher)
         methods = ENRICH_METHODS_BY_ENRICHER[enricher.class] || []
-        methods.each do |method|
-          send(method, enricher) if respond_to?(method)
-        end
+        methods.each { |method| send(method, enricher) if respond_to?(method) }
       end
 
       private
 
+      def ipinfo
+        @ipinfo ||= Enrichers::IPInfo.new
+      end
+
+      def shodan
+        @shodan ||= Enrichers::Shodan.new
+      end
+
       def normalize_as_domain(url_or_domain)
         return url_or_domain if data_type == "domain"
 

data/lib/mihari/models/rule.rb

@@ -66,8 +66,7 @@ module Mihari
         # @return [Mihari::Models::Rule]
         #
         def build_relation(filter)
-          relation = self
-          relation = relation.includes(alerts: :tags)
+          relation = includes(alerts: :tags)
 
           relation = relation.where(alerts: { tags: { name: filter.tag_name } }) if filter.tag_name
 

data/lib/mihari/rule.rb

@@ -113,15 +113,15 @@ module Mihari
       analyzers.flat_map do |analyzer|
         # @type [Dry::Monads::Result::Success<Array<Mihari::Models::Artifact>>, Dry::Monads::Result::Failure]
         result = analyzer.result
-
-        if result.failure?
-          raise result.failure unless analyzer.ignore_error?
-        else
+        case result
+        when Success
           artifacts = result.value!
           artifacts.map do |artifact|
             artifact.rule_id = id
             artifact
           end
+        else
+          raise result.failure unless analyzer.ignore_error?
         end
       end.compact
     end
@@ -177,8 +177,14 @@ module Mihari
       results = Parallel.map(emitters) { |emitter| emitter.result enriched_artifacts }
       results.zip(emitters).map do |result_and_emitter|
         result, emitter = result_and_emitter
-        Mihari.logger.info "Emission by #{emitter.class} is failed: #{result.failure}" if result.failure?
-        Mihari.logger.info "Emission by #{emitter.class} is succeeded" if result.success?
+
+        case result
+        when Success
+          Mihari.logger.info "Emission by #{emitter.class} succeed"
+        else
+          Mihari.logger.info "Emission by #{emitter.class} failed: #{result.failure}"
+        end
+
         result.value_or nil
       end.compact
     end
@@ -289,8 +295,7 @@ module Mihari
       @analyzers ||= queries.map do |query_params|
         analyzer_name = query_params[:analyzer]
         klass = get_analyzer_class(analyzer_name)
-        klass.from_query(query_params)
-      end.map do |analyzer|
+        analyzer = klass.from_query(query_params)
         analyzer.validate_configuration!
         analyzer
       end
@@ -320,8 +325,7 @@ module Mihari
         %i[emitter options].each { |key| params.delete key }
 
         klass = get_emitter_class(name)
-        klass.new(rule: self, options: options, **params)
-      end.map do |emitter|
+        emitter = klass.new(rule: self, options: options, **params)
         emitter.validate_configuration!
         emitter
       end