mihari 3.12.0 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5a17c14dffe68b2f82316858615371036996a3a2b6d31d3c6a5c431294c38634
-  data.tar.gz: 97afaf2f8b20985b0908cc1fe5778fe75ab7d0e1bf13b990a7b4165c92843604
+  metadata.gz: 1a42a8897840d2f2268f88c6734c1633642256ef08667c39f45f5d03ac13874f
+  data.tar.gz: 89bdd5aa38f833e3158464915ec46c05e8c0bb6f17d34fe397d1590c59a57d7c
 SHA512:
-  metadata.gz: 418d7f278536e245196723d2ebbe0d99934b0db3278e8f8178470241c67f25e05d0aacf7bab2b766ce69d0589a79e87eca554a91fe4cfae5cfa12cd52d97fb61
-  data.tar.gz: 9e90193273ee40c9956a138c2c73b713eb610a3ebdc1c9def73a6d354c4124a97e6842e438004956b3edc3206dfee8bae5771a86e964b59a2a7065a1f910891f
+  metadata.gz: f62927fdb96eabffd99106bf03178e6eb573ee5e58b8fa2bb94b6b6b2ea898324fd65656bbb128aab67196ad03da959603dd73108c8840e883560d673c8afa5f
+  data.tar.gz: 2e752fa7ab93691ad6c38ec534af3bab14e27f2b66127dc00fc016f26554ea29fc2c743d2bdaef93d85e816d8a47e54d2d41bb6dafaabf711cfb2c833c3ff0ff

data/lib/mihari/analyzers/base.rb

@@ -1,8 +1,5 @@
 # frozen_string_literal: true
 
-require "dry-initializer"
-require "parallel"
-
 module Mihari
   module Analyzers
     class Base
@@ -10,6 +7,7 @@ module Mihari
 
       include Mixins::AutonomousSystem
       include Mixins::Configurable
+      include Mixins::Database
       include Mixins::Retriable
 
       attr_accessor :ignore_old_artifacts, :ignore_threshold
@@ -52,10 +50,12 @@ module Mihari
       # @return [nil]
       #
       def run
-        set_enriched_artifacts
+        with_db_connection do
+          set_enriched_artifacts
 
-        Parallel.each(valid_emitters) do |emitter|
-          run_emitter emitter
+          Parallel.each(valid_emitters) do |emitter|
+            run_emitter emitter
+          end
         end
       end
 

data/lib/mihari/analyzers/binaryedge.rb

@@ -6,9 +6,6 @@ module Mihari
   module Analyzers
     class BinaryEdge < Base
       param :query
-      option :title, default: proc { "BinaryEdge search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
 
       option :interval, default: proc { 0 }
 
@@ -19,9 +16,10 @@ module Mihari
         results.map do |result|
           events = result["events"] || []
           events.filter_map do |event|
-            event.dig "target", "ip"
+            data = event.dig("target", "ip")
+            data.nil? ? nil : Artifact.new(data: data, source: source, metadata: event)
           end
-        end.flatten.compact.uniq
+        end.flatten
       end
 
       private

data/lib/mihari/analyzers/censys.rb

@@ -6,9 +6,6 @@ module Mihari
   module Analyzers
     class Censys < Base
       param :query
-      option :title, default: proc { "Censys search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
 
       option :interval, default: proc { 0 }
 
@@ -77,6 +74,7 @@ module Mihari
         Artifact.new(
           data: hit.ip,
           source: source,
+          metadata: hit.metadata,
           autonomous_system: as,
           geolocation: geolocation
         )

data/lib/mihari/analyzers/circl.rb

@@ -8,9 +8,6 @@ module Mihari
       include Mixins::Refang
 
       param :query
-      option :title, default: proc { "CIRCL passive DNS/SSL search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
 
       attr_reader :type
 

data/lib/mihari/analyzers/crtsh.rb

@@ -6,15 +6,17 @@ module Mihari
   module Analyzers
     class Crtsh < Base
       param :query
-      option :title, default: proc { "crt.sh search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
+
       option :exclude_expired, default: proc { true }
 
       def artifacts
         results = search
-        name_values = results.filter_map { |result| result["name_value"] }
-        name_values.map(&:lines).flatten.uniq.map(&:chomp)
+        results.map do |result|
+          values = result["name_value"].to_s.lines.map(&:chomp)
+          values.map do |value|
+            Artifact.new(data: value, source: source, metadata: result)
+          end
+        end.flatten
       end
 
       private

data/lib/mihari/analyzers/dnpedia.rb

@@ -6,8 +6,7 @@ module Mihari
   module Analyzers
     class DNPedia < Base
       param :query
-      option :title, default: proc { "DNPedia domain search" }
-      option :description, default: proc { "query = #{query}" }
+
       option :tags, default: proc { [] }
 
       def artifacts
@@ -23,13 +22,14 @@ module Mihari
       #
       # Search
       #
-      # @return [Array<String>]
+      # @return [Array<Mihari::Artifact>]
       #
       def search
         res = api.search(query)
         rows = res["rows"] || []
         rows.map do |row|
-          [row["name"], row["zoneid"]].join(".")
+          data = [row["name"], row["zoneid"]].join(".")
+          Artifact.new(data: data, source: source, metadata: row)
         end
       end
     end

data/lib/mihari/analyzers/dnstwister.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
 require "dnstwister"
-require "resolv"
-require "parallel"
 
 module Mihari
   module Analyzers
@@ -10,8 +8,7 @@ module Mihari
       include Mixins::Refang
 
       param :query
-      option :title, default: proc { "dnstwister domain search" }
-      option :description, default: proc { "query = #{query}" }
+
       option :tags, default: proc { [] }
 
       attr_reader :type

data/lib/mihari/analyzers/feed.rb

@@ -7,9 +7,6 @@ module Mihari
   module Analyzers
     class Feed < Base
       param :query
-      option :title, default: proc { "Feed" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
 
       option :http_request_method, default: proc { "GET" }
       option :http_request_headers, default: proc { {} }

data/lib/mihari/analyzers/greynoise.rb

@@ -6,9 +6,6 @@ module Mihari
   module Analyzers
     class GreyNoise < Base
       param :query
-      option :title, default: proc { "GreyNoise search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
 
       def artifacts
         res = Structs::GreyNoise::Response.from_dynamic!(search)
@@ -56,6 +53,7 @@ module Mihari
         Artifact.new(
           data: datum.ip,
           source: source,
+          metadata: datum.metadata_,
           autonomous_system: as,
           geolocation: geolocation
         )

data/lib/mihari/analyzers/onyphe.rb

@@ -7,9 +7,6 @@ module Mihari
   module Analyzers
     class Onyphe < Base
       param :query
-      option :title, default: proc { "Onyphe search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
 
       option :interval, default: proc { 0 }
 
@@ -89,6 +86,7 @@ module Mihari
         Artifact.new(
           data: result.ip,
           source: source,
+          metadata: result.metadata,
           autonomous_system: as,
           geolocation: geolocation
         )

data/lib/mihari/analyzers/otx.rb

@@ -8,9 +8,6 @@ module Mihari
       include Mixins::Refang
 
       param :query
-      option :title, default: proc { "OTX search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
 
       attr_reader :type
 

data/lib/mihari/analyzers/passivetotal.rb

@@ -8,9 +8,6 @@ module Mihari
       include Mixins::Refang
 
       param :query
-      option :title, default: proc { "PassiveTotal search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
 
       attr_reader :type
 
@@ -75,27 +72,29 @@ module Mihari
       #
       # Reverse whois search
       #
-      # @return [Array<String>]
+      # @return [Array<Mihari::Artifact>]
       #
       def reverse_whois_search
         res = api.whois.search(query: query, field: "email")
         results = res["results"] || []
         results.map do |result|
-          result["domain"]
-        end.flatten.compact.uniq
+          data = result["domain"]
+          Artifact.new(data: data, source: source, metadata: result)
+        end.flatten
       end
 
       #
       # Passive SSL search
       #
-      # @return [Array<String>]
+      # @return [Array<Mihari::Artifact>]
       #
       def ssl_search
         res = api.ssl.history(query)
         results = res["results"] || []
         results.map do |result|
-          result["ipAddresses"]
-        end.flatten.compact.uniq
+          data = result["ipAddresses"]
+          Artifact.new(data: data, source: source, metadata: result)
+        end.flatten
       end
     end
   end

data/lib/mihari/analyzers/pulsedive.rb

@@ -8,9 +8,6 @@ module Mihari
       include Mixins::Refang
 
       param :query
-      option :title, default: proc { "Pulsedive search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
 
       attr_reader :type
 
@@ -47,7 +44,7 @@ module Mihari
       #
       # Search
       #
-      # @return [Array<String>]
+      # @return [Array<Mihari::Artifact>]
       #
       def search
         raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
@@ -57,7 +54,12 @@ module Mihari
 
         properties = api.indicator.get_properties_by_id(iid)
         (properties["dns"] || []).filter_map do |property|
-          property["value"] if ["A", "PTR"].include?(property["name"])
+          if ["A", "PTR"].include?(property["name"])
+            nil
+          else
+            data = property["value"]
+            Artifact.new(data: data, source: source, metadata: property)
+          end
         end
       end
     end

data/lib/mihari/analyzers/rule.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "uuidtools"
-
 module Mihari
   module Analyzers
     ANALYZER_TO_CLASS = {
@@ -31,13 +29,14 @@ module Mihari
     }.freeze
 
     class Rule < Base
-      include Mihari::Mixins::DisallowedDataValue
+      include Mixins::DisallowedDataValue
+      include Mixins::Rule
 
       option :title
       option :description
       option :queries
 
-      option :id, default: proc {}
+      option :id, default: proc { "" }
       option :tags, default: proc { [] }
       option :allowed_data_types, default: proc { ALLOWED_DATA_TYPES }
       option :disallowed_data_values, default: proc { [] }
@@ -47,7 +46,7 @@ module Mihari
       def initialize(**kwargs)
         super(**kwargs)
 
-        @source = id || UUIDTools::UUID.md5_create(UUIDTools::UUID_URL_NAMESPACE, title + description).to_s
+        @source = id
 
         validate_analyzer_configurations
       end
@@ -69,7 +68,7 @@ module Mihari
           # set interval in the top level
           options = params[:options] || {}
           interval = options[:interval]
-          params[:interval] = interval
+          params[:interval] = interval if interval
 
           analyzer = klass.new(query, **params)
 

data/lib/mihari/analyzers/securitytrails.rb

@@ -8,9 +8,6 @@ module Mihari
       include Mixins::Refang
 
       param :query
-      option :title, default: proc { "SecurityTrails search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
 
       attr_reader :type
 
@@ -47,7 +44,7 @@ module Mihari
       #
       # IP/domain/mail search
       #
-      # @return [Array<String>]
+      # @return [Array<String>, Array<Mihari::Artifact>]
       #
       def search
         case type
@@ -78,12 +75,15 @@ module Mihari
       #
       # IP search
       #
-      # @return [Array<String>]
+      # @return [Array<Mihari::Artifact>]
       #
       def ip_search
         result = api.domains.search(filter: { ipv4: query })
         records = result["records"] || []
-        records.filter_map { |record| record["hostname"] }.uniq
+        records.filter_map do |record|
+          data = record["hostname"]
+          Artifact.new(data: data, source: source, metadata: record)
+        end
       end
 
       #
@@ -94,7 +94,10 @@ module Mihari
       def mail_search
         result = api.domains.search(filter: { whois_email: query })
         records = result["records"] || []
-        records.filter_map { |record| record["hostname"] }.uniq
+        records.filter_map do |record|
+          data = record["hostname"]
+          Artifact.new(data: data, source: source, metadata: record)
+        end
       end
     end
   end

data/lib/mihari/analyzers/shodan.rb

@@ -6,9 +6,6 @@ module Mihari
   module Analyzers
     class Shodan < Base
       param :query
-      option :title, default: proc { "Shodan search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
 
       option :interval, default: proc { 0 }
 
@@ -20,7 +17,7 @@ module Mihari
         results.map do |result|
           matches = result.matches || []
           matches.map { |match| build_artifact match }
-        end.flatten.compact.uniq(&:data)
+        end.flatten.uniq(&:data)
       end
 
       private
@@ -98,6 +95,7 @@ module Mihari
         Artifact.new(
           data: match.ip_str,
           source: source,
+          metadata: match.metadata,
           autonomous_system: as,
           geolocation: geolocation
         )

data/lib/mihari/analyzers/spyse.rb

@@ -1,16 +1,13 @@
 # frozen_string_literal: true
 
 require "spyse"
-require "json"
 
 module Mihari
   module Analyzers
     class Spyse < Base
       param :query
-      option :title, default: proc { "Spyse search" }
-      option :description, default: proc { "query = #{query}" }
+
       option :type, default: proc { "domain" }
-      option :tags, default: proc { [] }
 
       def artifacts
         search || []
@@ -42,33 +39,35 @@ module Mihari
       #
       # Domain search
       #
-      # @return [Array<String>]
+      # @return [Array<Mihari::Artifact>]
       #
       def domain_search
         res = api.domain.search(search_params, limit: 100)
         items = res.dig("data", "items") || []
         items.map do |item|
-          item["name"]
-        end.uniq.compact
+          data = item["name"]
+          Artifact.new(data: data, source: source, metadata: item)
+        end
       end
 
       #
       # IP search
       #
-      # @return [Array<String>]
+      # @return [Array<Mihari::Artifact>]
       #
       def ip_search
         res = api.ip.search(search_params, limit: 100)
         items = res.dig("data", "items") || []
         items.map do |item|
-          item["ip"]
-        end.uniq.compact
+          data = item["ip"]
+          Artifact.new(data: data, source: source, metadata: item)
+        end
       end
 
       #
       # IP/domain search
       #
-      # @return [Array<String>]
+      # @return [Array<Mihari::Artifact>]
       #
       def search
         case type

data/lib/mihari/analyzers/urlscan.rb

@@ -6,9 +6,7 @@ module Mihari
   module Analyzers
     class Urlscan < Base
       param :query
-      option :title, default: proc { "urlscan search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
+
       option :allowed_data_types, default: proc { SUPPORTED_DATA_TYPES }
 
       option :interval, default: proc { 0 }
@@ -29,9 +27,10 @@ module Mihari
         allowed_data_types.map do |type|
           results.filter_map do |result|
             page = result.page
-            page.send(type.to_sym)
-          end.uniq
-        end.flatten.compact
+            data = page.send(type.to_sym)
+            data.nil? ? nil : Artifact.new(data: data, source: source, metadata: result)
+          end
+        end.flatten
       end
 
       private

data/lib/mihari/analyzers/virustotal.rb

@@ -8,9 +8,6 @@ module Mihari
       include Mixins::Refang
 
       param :query
-      option :title, default: proc { "VirusTotal search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
 
       attr_reader :type
 
@@ -47,7 +44,7 @@ module Mihari
       #
       # Search
       #
-      # @return [Array<String>]
+      # @return [Array<Mihari::Artifact>]
       #
       def search
         case type
@@ -63,28 +60,30 @@ module Mihari
       #
       # Domain search
       #
-      # @return [Array<String>]
+      # @return [Array<Mihari::Artifact>]
       #
       def domain_search
         res = api.domain.resolutions(query)
 
         data = res["data"] || []
         data.filter_map do |item|
-          item.dig("attributes", "ip_address")
-        end.uniq
+          data = item.dig("attributes", "ip_address")
+          data.nil? ? nil : Artifact.new(data: data, source: source, metadata: item)
+        end
       end
 
       #
       # IP search
       #
-      # @return [Array<String>]
+      # @return [Array<Mihari::Artifact>]
       #
       def ip_search
         res = api.ip_address.resolutions(query)
 
         data = res["data"] || []
         data.filter_map do |item|
-          item.dig("attributes", "host_name")
+          data = item.dig("attributes", "host_name")
+          Artifact.new(data: data, source: source, metadata: item)
         end.uniq
       end
     end

data/lib/mihari/analyzers/virustotal_intelligence.rb

@@ -6,9 +6,6 @@ module Mihari
   module Analyzers
     class VirusTotalIntelligence < Base
       param :query
-      option :title, default: proc { "VirusTotal Intelligence search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
 
       option :interval, default: proc { 0 }
 
@@ -21,8 +18,10 @@ module Mihari
       def artifacts
         responses = search_witgh_cursor
         responses.map do |response|
-          response.data.map(&:value)
-        end.flatten.compact.uniq
+          response.data.map do |datum|
+            Artifact.new(data: datum.value, source: source, metadata: datum.metadata)
+          end
+        end.flatten
       end
 
       private

data/lib/mihari/analyzers/zoomeye.rb

@@ -6,9 +6,7 @@ module Mihari
   module Analyzers
     class ZoomEye < Base
       param :query
-      option :title, default: proc { "ZoomEye search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
+
       option :type, default: proc { "host" }
 
       option :interval, default: proc { 0 }
@@ -50,13 +48,14 @@ module Mihari
       #
       # @param [Array<Hash>] responses
       #
-      # @return [Array<String>]
+      # @return [Array<Mihari::Artifact>]
       #
       def convert_responses(responses)
         responses.map do |res|
           matches = res["matches"] || []
           matches.map do |match|
-            match["ip"]
+            data = match["ip"]
+            Artifact.new(data: data, source: source, metadata: match)
           end
         end.flatten.compact.uniq
       end

data/lib/mihari/cli/base.rb

@@ -1,15 +1,10 @@
 # frozen_string_literal: true
 
-require "thor"
-
-require "mihari/mixins/hash"
-
 require "mihari/cli/mixins/utils"
 
 module Mihari
   module CLI
     class Base < Thor
-      include Mihari::Mixins::Hash
       include Mixins::Utils
 
       class << self

data/lib/mihari/cli/init.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "thor"
-
 require "mihari/commands/init"
 
 module Mihari

data/lib/mihari/cli/main.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "thor"
+
 # Commands
 require "mihari/commands/search"
 require "mihari/commands/web"
@@ -7,7 +9,6 @@ require "mihari/commands/web"
 # CLIs
 require "mihari/cli/base"
 
-require "mihari/cli/analyzer"
 require "mihari/cli/init"
 require "mihari/cli/validator"
 
@@ -17,13 +18,10 @@ module Mihari
       include Mihari::Commands::Search
       include Mihari::Commands::Web
 
-      desc "analyze", "Sub commands to run an analyzer"
-      subcommand "analyze", Analyzer
-
-      desc "init", "Sub commands to initialize config & rule"
+      desc "init", "Sub commands to initialize a rule"
       subcommand "init", Initialization
 
-      desc "validate", "Sub commands to validate format of config & rule"
+      desc "validate", "Sub commands to validate format of a rule"
       subcommand "validate", Validator
     end
   end