mihari 3.12.0 → 4.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5a17c14dffe68b2f82316858615371036996a3a2b6d31d3c6a5c431294c38634
-  data.tar.gz: 97afaf2f8b20985b0908cc1fe5778fe75ab7d0e1bf13b990a7b4165c92843604
+  metadata.gz: c9d5f66d8c2bc30eeb644d6841662778e5457db184e21a35d9bc689cbf653d25
+  data.tar.gz: 64edb36cf057854f4216ac51158dfa3e2b2876bb89a81c9441e38a4015c9bc78
 SHA512:
-  metadata.gz: 418d7f278536e245196723d2ebbe0d99934b0db3278e8f8178470241c67f25e05d0aacf7bab2b766ce69d0589a79e87eca554a91fe4cfae5cfa12cd52d97fb61
-  data.tar.gz: 9e90193273ee40c9956a138c2c73b713eb610a3ebdc1c9def73a6d354c4124a97e6842e438004956b3edc3206dfee8bae5771a86e964b59a2a7065a1f910891f
+  metadata.gz: 54eb1b87c28c9af6a2e045ecc1c4bacb575353fe7a7eaa7f9c5ba2a4cea0a15b46342707b444b3a65f9c14317c6b19c58c8085feb653f6167a57c7f3231ce284
+  data.tar.gz: b67ece0bb87d24ca83773a1c50b80560b46505a069f26a9c4c0ffe09105f2fe05ddafb4cfd5d0e1e7b1603ae5f1a11ea1d7b161971c37207c07869acb49e9b6c

data/Rakefile

@@ -7,3 +7,8 @@ require "standard/rake"
 RSpec::Core::RakeTask.new(:spec)
 
 task default: :spec
+
+desc "run rackup (via rerun)"
+task :rackup do
+  sh "rerun --pattern '{Gemfile.lock,lib/**/*.rb,lib/*.rb}' -- rackup config.ru"
+end

data/config.ru

@@ -1,4 +1,3 @@
-# bundle exec rerun -- rackup config.ru
 require "./lib/mihari"
 
 # set rack env as development

data/lib/mihari/analyzers/base.rb

@@ -1,8 +1,5 @@
 # frozen_string_literal: true
 
-require "dry-initializer"
-require "parallel"
-
 module Mihari
   module Analyzers
     class Base
@@ -10,6 +7,7 @@ module Mihari
 
       include Mixins::AutonomousSystem
       include Mixins::Configurable
+      include Mixins::Database
       include Mixins::Retriable
 
       attr_accessor :ignore_old_artifacts, :ignore_threshold
@@ -52,10 +50,12 @@ module Mihari
       # @return [nil]
       #
       def run
-        set_enriched_artifacts
+        with_db_connection do
+          set_enriched_artifacts
 
-        Parallel.each(valid_emitters) do |emitter|
-          run_emitter emitter
+          Parallel.each(valid_emitters) do |emitter|
+            run_emitter emitter
+          end
         end
       end
 

data/lib/mihari/analyzers/binaryedge.rb

@@ -6,9 +6,6 @@ module Mihari
   module Analyzers
     class BinaryEdge < Base
       param :query
-      option :title, default: proc { "BinaryEdge search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
 
       option :interval, default: proc { 0 }
 
@@ -19,9 +16,10 @@ module Mihari
         results.map do |result|
           events = result["events"] || []
           events.filter_map do |event|
-            event.dig "target", "ip"
+            data = event.dig("target", "ip")
+            data.nil? ? nil : Artifact.new(data: data, source: source, metadata: event)
           end
-        end.flatten.compact.uniq
+        end.flatten
       end
 
       private

data/lib/mihari/analyzers/censys.rb

@@ -6,9 +6,6 @@ module Mihari
   module Analyzers
     class Censys < Base
       param :query
-      option :title, default: proc { "Censys search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
 
       option :interval, default: proc { 0 }
 
@@ -77,6 +74,7 @@ module Mihari
         Artifact.new(
           data: hit.ip,
           source: source,
+          metadata: hit.metadata,
           autonomous_system: as,
           geolocation: geolocation
         )

data/lib/mihari/analyzers/circl.rb

@@ -8,9 +8,6 @@ module Mihari
       include Mixins::Refang
 
       param :query
-      option :title, default: proc { "CIRCL passive DNS/SSL search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
 
       attr_reader :type
 

data/lib/mihari/analyzers/crtsh.rb

@@ -6,15 +6,17 @@ module Mihari
   module Analyzers
     class Crtsh < Base
       param :query
-      option :title, default: proc { "crt.sh search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
+
       option :exclude_expired, default: proc { true }
 
       def artifacts
         results = search
-        name_values = results.filter_map { |result| result["name_value"] }
-        name_values.map(&:lines).flatten.uniq.map(&:chomp)
+        results.map do |result|
+          values = result["name_value"].to_s.lines.map(&:chomp)
+          values.map do |value|
+            Artifact.new(data: value, source: source, metadata: result)
+          end
+        end.flatten
       end
 
       private

data/lib/mihari/analyzers/dnpedia.rb

@@ -6,8 +6,7 @@ module Mihari
   module Analyzers
     class DNPedia < Base
       param :query
-      option :title, default: proc { "DNPedia domain search" }
-      option :description, default: proc { "query = #{query}" }
+
       option :tags, default: proc { [] }
 
       def artifacts
@@ -23,13 +22,14 @@ module Mihari
       #
       # Search
       #
-      # @return [Array<String>]
+      # @return [Array<Mihari::Artifact>]
       #
       def search
         res = api.search(query)
         rows = res["rows"] || []
         rows.map do |row|
-          [row["name"], row["zoneid"]].join(".")
+          data = [row["name"], row["zoneid"]].join(".")
+          Artifact.new(data: data, source: source, metadata: row)
         end
       end
     end

data/lib/mihari/analyzers/dnstwister.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
 require "dnstwister"
-require "resolv"
-require "parallel"
 
 module Mihari
   module Analyzers
@@ -10,8 +8,7 @@ module Mihari
       include Mixins::Refang
 
       param :query
-      option :title, default: proc { "dnstwister domain search" }
-      option :description, default: proc { "query = #{query}" }
+
       option :tags, default: proc { [] }
 
       attr_reader :type

data/lib/mihari/analyzers/feed.rb

@@ -7,9 +7,6 @@ module Mihari
   module Analyzers
     class Feed < Base
       param :query
-      option :title, default: proc { "Feed" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
 
       option :http_request_method, default: proc { "GET" }
       option :http_request_headers, default: proc { {} }

data/lib/mihari/analyzers/greynoise.rb

@@ -6,9 +6,6 @@ module Mihari
   module Analyzers
     class GreyNoise < Base
       param :query
-      option :title, default: proc { "GreyNoise search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
 
       def artifacts
         res = Structs::GreyNoise::Response.from_dynamic!(search)
@@ -56,6 +53,7 @@ module Mihari
         Artifact.new(
           data: datum.ip,
           source: source,
+          metadata: datum.metadata_,
           autonomous_system: as,
           geolocation: geolocation
         )

data/lib/mihari/analyzers/onyphe.rb

@@ -7,9 +7,6 @@ module Mihari
   module Analyzers
     class Onyphe < Base
       param :query
-      option :title, default: proc { "Onyphe search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
 
       option :interval, default: proc { 0 }
 
@@ -89,6 +86,7 @@ module Mihari
         Artifact.new(
           data: result.ip,
           source: source,
+          metadata: result.metadata,
           autonomous_system: as,
           geolocation: geolocation
         )

data/lib/mihari/analyzers/otx.rb

@@ -8,9 +8,6 @@ module Mihari
       include Mixins::Refang
 
       param :query
-      option :title, default: proc { "OTX search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
 
       attr_reader :type
 

data/lib/mihari/analyzers/passivetotal.rb

@@ -8,9 +8,6 @@ module Mihari
       include Mixins::Refang
 
       param :query
-      option :title, default: proc { "PassiveTotal search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
 
       attr_reader :type
 
@@ -75,27 +72,29 @@ module Mihari
       #
       # Reverse whois search
       #
-      # @return [Array<String>]
+      # @return [Array<Mihari::Artifact>]
       #
       def reverse_whois_search
         res = api.whois.search(query: query, field: "email")
         results = res["results"] || []
         results.map do |result|
-          result["domain"]
-        end.flatten.compact.uniq
+          data = result["domain"]
+          Artifact.new(data: data, source: source, metadata: result)
+        end.flatten
       end
 
       #
       # Passive SSL search
       #
-      # @return [Array<String>]
+      # @return [Array<Mihari::Artifact>]
       #
       def ssl_search
         res = api.ssl.history(query)
         results = res["results"] || []
         results.map do |result|
-          result["ipAddresses"]
-        end.flatten.compact.uniq
+          data = result["ipAddresses"]
+          Artifact.new(data: data, source: source, metadata: result)
+        end.flatten
       end
     end
   end

data/lib/mihari/analyzers/pulsedive.rb

@@ -8,9 +8,6 @@ module Mihari
       include Mixins::Refang
 
       param :query
-      option :title, default: proc { "Pulsedive search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
 
       attr_reader :type
 
@@ -47,7 +44,7 @@ module Mihari
       #
       # Search
       #
-      # @return [Array<String>]
+      # @return [Array<Mihari::Artifact>]
       #
       def search
         raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
@@ -57,7 +54,12 @@ module Mihari
 
         properties = api.indicator.get_properties_by_id(iid)
         (properties["dns"] || []).filter_map do |property|
-          property["value"] if ["A", "PTR"].include?(property["name"])
+          if ["A", "PTR"].include?(property["name"])
+            nil
+          else
+            data = property["value"]
+            Artifact.new(data: data, source: source, metadata: property)
+          end
         end
       end
     end

data/lib/mihari/analyzers/rule.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "uuidtools"
-
 module Mihari
   module Analyzers
     ANALYZER_TO_CLASS = {
@@ -31,13 +29,14 @@ module Mihari
     }.freeze
 
     class Rule < Base
-      include Mihari::Mixins::DisallowedDataValue
+      include Mixins::DisallowedDataValue
+      include Mixins::Rule
 
       option :title
       option :description
       option :queries
 
-      option :id, default: proc {}
+      option :id, default: proc { "" }
       option :tags, default: proc { [] }
       option :allowed_data_types, default: proc { ALLOWED_DATA_TYPES }
       option :disallowed_data_values, default: proc { [] }
@@ -47,7 +46,7 @@ module Mihari
       def initialize(**kwargs)
         super(**kwargs)
 
-        @source = id || UUIDTools::UUID.md5_create(UUIDTools::UUID_URL_NAMESPACE, title + description).to_s
+        @source = id
 
         validate_analyzer_configurations
       end
@@ -69,7 +68,7 @@ module Mihari
           # set interval in the top level
           options = params[:options] || {}
           interval = options[:interval]
-          params[:interval] = interval
+          params[:interval] = interval if interval
 
           analyzer = klass.new(query, **params)
 

data/lib/mihari/analyzers/securitytrails.rb

@@ -8,9 +8,6 @@ module Mihari
       include Mixins::Refang
 
       param :query
-      option :title, default: proc { "SecurityTrails search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
 
       attr_reader :type
 
@@ -47,7 +44,7 @@ module Mihari
       #
       # IP/domain/mail search
       #
-      # @return [Array<String>]
+      # @return [Array<String>, Array<Mihari::Artifact>]
       #
       def search
         case type
@@ -78,12 +75,15 @@ module Mihari
       #
       # IP search
       #
-      # @return [Array<String>]
+      # @return [Array<Mihari::Artifact>]
       #
       def ip_search
         result = api.domains.search(filter: { ipv4: query })
         records = result["records"] || []
-        records.filter_map { |record| record["hostname"] }.uniq
+        records.filter_map do |record|
+          data = record["hostname"]
+          Artifact.new(data: data, source: source, metadata: record)
+        end
       end
 
       #
@@ -94,7 +94,10 @@ module Mihari
       def mail_search
         result = api.domains.search(filter: { whois_email: query })
         records = result["records"] || []
-        records.filter_map { |record| record["hostname"] }.uniq
+        records.filter_map do |record|
+          data = record["hostname"]
+          Artifact.new(data: data, source: source, metadata: record)
+        end
       end
     end
   end

data/lib/mihari/analyzers/shodan.rb

@@ -6,9 +6,6 @@ module Mihari
   module Analyzers
     class Shodan < Base
       param :query
-      option :title, default: proc { "Shodan search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
 
       option :interval, default: proc { 0 }
 
@@ -19,8 +16,8 @@ module Mihari
         results = results.map { |result| Structs::Shodan::Result.from_dynamic!(result) }
         results.map do |result|
           matches = result.matches || []
-          matches.map { |match| build_artifact match }
-        end.flatten.compact.uniq(&:data)
+          matches.map { |match| build_artifact(match, matches) }
+        end.flatten.uniq(&:data)
       end
 
       private
@@ -76,14 +73,27 @@ module Mihari
         responses
       end
 
+      #
+      # Collect metadata from matches
+      #
+      # @param [Array<Structs::Shodan::Match>] matches
+      # @param [String] ip
+      #
+      # @return [Array<Hash>]
+      #
+      def collect_metadata_by_ip(matches, ip)
+        matches.select { |match| match.ip_str == ip }.map(&:metadata)
+      end
+
       #
       # Build an artifact from a Shodan search API response
       #
       # @param [Structs::Shodan::Match] match
+      # @param [Array<Structs::Shodan::Match>] matches
       #
       # @return [Artifact]
       #
-      def build_artifact(match)
+      def build_artifact(match, matches)
         as = nil
         as = AutonomousSystem.new(asn: normalize_asn(match.asn)) unless match.asn.nil?
 
@@ -95,9 +105,12 @@ module Mihari
           )
         end
 
+        metadata = collect_metadata_by_ip(matches, match.ip_str)
+
         Artifact.new(
           data: match.ip_str,
           source: source,
+          metadata: metadata,
           autonomous_system: as,
           geolocation: geolocation
         )

data/lib/mihari/analyzers/spyse.rb

@@ -1,16 +1,13 @@
 # frozen_string_literal: true
 
 require "spyse"
-require "json"
 
 module Mihari
   module Analyzers
     class Spyse < Base
       param :query
-      option :title, default: proc { "Spyse search" }
-      option :description, default: proc { "query = #{query}" }
+
       option :type, default: proc { "domain" }
-      option :tags, default: proc { [] }
 
       def artifacts
         search || []
@@ -42,33 +39,35 @@ module Mihari
       #
       # Domain search
       #
-      # @return [Array<String>]
+      # @return [Array<Mihari::Artifact>]
       #
       def domain_search
         res = api.domain.search(search_params, limit: 100)
         items = res.dig("data", "items") || []
         items.map do |item|
-          item["name"]
-        end.uniq.compact
+          data = item["name"]
+          Artifact.new(data: data, source: source, metadata: item)
+        end
       end
 
       #
       # IP search
       #
-      # @return [Array<String>]
+      # @return [Array<Mihari::Artifact>]
       #
       def ip_search
         res = api.ip.search(search_params, limit: 100)
         items = res.dig("data", "items") || []
         items.map do |item|
-          item["ip"]
-        end.uniq.compact
+          data = item["ip"]
+          Artifact.new(data: data, source: source, metadata: item)
+        end
       end
 
       #
       # IP/domain search
       #
-      # @return [Array<String>]
+      # @return [Array<Mihari::Artifact>]
       #
       def search
         case type

data/lib/mihari/analyzers/urlscan.rb

@@ -6,9 +6,7 @@ module Mihari
   module Analyzers
     class Urlscan < Base
       param :query
-      option :title, default: proc { "urlscan search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
+
       option :allowed_data_types, default: proc { SUPPORTED_DATA_TYPES }
 
       option :interval, default: proc { 0 }
@@ -29,9 +27,10 @@ module Mihari
         allowed_data_types.map do |type|
           results.filter_map do |result|
             page = result.page
-            page.send(type.to_sym)
-          end.uniq
-        end.flatten.compact
+            data = page.send(type.to_sym)
+            data.nil? ? nil : Artifact.new(data: data, source: source, metadata: result)
+          end
+        end.flatten
       end
 
       private

data/lib/mihari/analyzers/virustotal.rb

@@ -8,9 +8,6 @@ module Mihari
       include Mixins::Refang
 
       param :query
-      option :title, default: proc { "VirusTotal search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
 
       attr_reader :type
 
@@ -47,7 +44,7 @@ module Mihari
       #
       # Search
       #
-      # @return [Array<String>]
+      # @return [Array<Mihari::Artifact>]
       #
       def search
         case type
@@ -63,28 +60,30 @@ module Mihari
       #
       # Domain search
       #
-      # @return [Array<String>]
+      # @return [Array<Mihari::Artifact>]
       #
       def domain_search
         res = api.domain.resolutions(query)
 
         data = res["data"] || []
         data.filter_map do |item|
-          item.dig("attributes", "ip_address")
-        end.uniq
+          data = item.dig("attributes", "ip_address")
+          data.nil? ? nil : Artifact.new(data: data, source: source, metadata: item)
+        end
       end
 
       #
       # IP search
       #
-      # @return [Array<String>]
+      # @return [Array<Mihari::Artifact>]
       #
       def ip_search
         res = api.ip_address.resolutions(query)
 
         data = res["data"] || []
         data.filter_map do |item|
-          item.dig("attributes", "host_name")
+          data = item.dig("attributes", "host_name")
+          Artifact.new(data: data, source: source, metadata: item)
         end.uniq
       end
     end

data/lib/mihari/analyzers/virustotal_intelligence.rb

@@ -6,9 +6,6 @@ module Mihari
   module Analyzers
     class VirusTotalIntelligence < Base
       param :query
-      option :title, default: proc { "VirusTotal Intelligence search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
 
       option :interval, default: proc { 0 }
 
@@ -21,8 +18,10 @@ module Mihari
       def artifacts
         responses = search_witgh_cursor
         responses.map do |response|
-          response.data.map(&:value)
-        end.flatten.compact.uniq
+          response.data.map do |datum|
+            Artifact.new(data: datum.value, source: source, metadata: datum.metadata)
+          end
+        end.flatten
       end
 
       private

data/lib/mihari/analyzers/zoomeye.rb

@@ -6,9 +6,7 @@ module Mihari
   module Analyzers
     class ZoomEye < Base
       param :query
-      option :title, default: proc { "ZoomEye search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
+
       option :type, default: proc { "host" }
 
       option :interval, default: proc { 0 }
@@ -50,13 +48,14 @@ module Mihari
       #
       # @param [Array<Hash>] responses
       #
-      # @return [Array<String>]
+      # @return [Array<Mihari::Artifact>]
       #
       def convert_responses(responses)
         responses.map do |res|
           matches = res["matches"] || []
           matches.map do |match|
-            match["ip"]
+            data = match["ip"]
+            Artifact.new(data: data, source: source, metadata: match)
           end
         end.flatten.compact.uniq
       end