mihari 2.4.0 → 3.0.0

data/lib/mihari/analyzers/circl.rb

@@ -5,26 +5,29 @@ require "passive_circl"
 module Mihari
   module Analyzers
     class CIRCL < Base
-      attr_reader :title, :description, :tags
+      include Mixins::Refang
 
-      def initialize(query, title: nil, description: nil, tags: [])
-        super()
+      param :query
+      option :title, default: proc { "CIRCL passive DNS/SSL search" }
+      option :description, default: proc { "query = #{query}" }
+      option :tags, default: proc { [] }
 
-        @query = query
-        @type = TypeChecker.type(query)
+      attr_reader :type
+
+      def initialize(*args, **kwargs)
+        super
 
-        @title = title || "CIRCL passive lookup"
-        @description = description || "query = #{query}"
-        @tags = tags
+        @query = refang(query)
+        @type = TypeChecker.type(query)
       end
 
       def artifacts
-        lookup || []
+        search || []
       end
 
       private
 
-      def config_keys
+      def configuration_keys
         %w[circl_passive_password circl_passive_username]
       end
 
@@ -32,26 +35,26 @@ module Mihari
         @api ||= ::PassiveCIRCL::API.new(username: Mihari.config.circl_passive_username, password: Mihari.config.circl_passive_password)
       end
 
-      def lookup
+      def search
         case @type
         when "domain"
-          passive_dns_lookup
+          passive_dns_search
         when "hash"
-          passive_ssl_lookup
+          passive_ssl_search
         else
           raise InvalidInputError, "#{@query}(type: #{@type || "unknown"}) is not supported."
         end
       end
 
-      def passive_dns_lookup
+      def passive_dns_search
         results = api.dns.query(@query)
-        results.map do |result|
+        results.filter_map do |result|
           type = result["rrtype"]
           type == "A" ? result["rdata"] : nil
-        end.compact.uniq
+        end.uniq
       end
 
-      def passive_ssl_lookup
+      def passive_ssl_search
         result = api.ssl.cquery(@query)
         seen = result["seen"] || []
         seen.uniq

data/lib/mihari/analyzers/crtsh.rb

@@ -5,22 +5,15 @@ require "crtsh"
 module Mihari
   module Analyzers
     class Crtsh < Base
-      attr_reader :title, :description, :query, :tags, :exclude_expired
-
-      def initialize(query, title: nil, description: nil, tags: [], exclude_expired: nil)
-        super()
-
-        @query = query
-        @title = title || "crt.sh lookup"
-        @description = description || "query = #{query}"
-        @tags = tags
-
-        @exclude_expired = exclude_expired.nil? ? true : exclude_expired
-      end
+      param :query
+      option :title, default: proc { "crt.sh search" }
+      option :description, default: proc { "query = #{query}" }
+      option :tags, default: proc { [] }
+      option :exclude_expired, default: proc { true }
 
       def artifacts
         results = search
-        name_values = results.map { |result| result["name_value"] }.compact
+        name_values = results.filter_map { |result| result["name_value"] }
         name_values.map(&:lines).flatten.uniq.map(&:chomp)
       end
 

data/lib/mihari/analyzers/dnpedia.rb

@@ -5,19 +5,13 @@ require "dnpedia"
 module Mihari
   module Analyzers
     class DNPedia < Base
-      attr_reader :query, :title, :description, :tags
-
-      def initialize(query, title: nil, description: nil, tags: [])
-        super()
-
-        @query = query
-        @title = title || "DNPedia domain lookup"
-        @description = description || "query = #{query}"
-        @tags = tags
-      end
+      param :query
+      option :title, default: proc { "DNPedia domain search" }
+      option :description, default: proc { "query = #{query}" }
+      option :tags, default: proc { [] }
 
       def artifacts
-        lookup || []
+        search || []
       end
 
       private
@@ -26,7 +20,7 @@ module Mihari
         @api ||= ::DNPedia::API.new
       end
 
-      def lookup
+      def search
         res = api.search(query)
         rows = res["rows"] || []
         rows.map do |row|

data/lib/mihari/analyzers/dnstwister.rb

@@ -7,21 +7,24 @@ require "parallel"
 module Mihari
   module Analyzers
     class DNSTwister < Base
-      attr_reader :query, :type, :title, :description, :tags
+      include Mixins::Refang
 
-      def initialize(query, title: nil, description: nil, tags: [])
-        super()
+      param :query
+      option :title, default: proc { "dnstwister domain search" }
+      option :description, default: proc { "query = #{query}" }
+      option :tags, default: proc { [] }
 
-        @query = query
-        @type = TypeChecker.type(query)
+      attr_reader :type
+
+      def initialize(*args, **kwargs)
+        super
 
-        @title = title || "dnstwister domain lookup"
-        @description = description || "query = #{query}"
-        @tags = tags
+        @query = refang(query)
+        @type = TypeChecker.type(query)
       end
 
       def artifacts
-        lookup || []
+        search || []
       end
 
       private
@@ -41,7 +44,7 @@ module Mihari
         false
       end
 
-      def lookup
+      def search
         raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
 
         res = api.fuzz(query)

data/lib/mihari/analyzers/onyphe.rb

@@ -5,16 +5,10 @@ require "onyphe"
 module Mihari
   module Analyzers
     class Onyphe < Base
-      attr_reader :title, :description, :query, :tags
-
-      def initialize(query, title: nil, description: nil, tags: [])
-        super()
-
-        @query = query
-        @title = title || "Onyphe lookup"
-        @description = description || "query = #{query}"
-        @tags = tags
-      end
+      param :query
+      option :title, default: proc { "Onyphe search" }
+      option :description, default: proc { "query = #{query}" }
+      option :tags, default: proc { [] }
 
       def artifacts
         results = search
@@ -24,14 +18,14 @@ module Mihari
           result["results"]
         end.flatten.compact
 
-        flat_results.map { |result| result["ip"] }.compact.uniq
+        flat_results.filter_map { |result| result["ip"] }.uniq
       end
 
       private
 
       PAGE_SIZE = 10
 
-      def config_keys
+      def configuration_keys
         %w[onyphe_api_key]
       end
 

data/lib/mihari/analyzers/otx.rb

@@ -5,26 +5,29 @@ require "otx_ruby"
 module Mihari
   module Analyzers
     class OTX < Base
-      attr_reader :query, :type, :title, :description, :tags
+      include Mixins::Refang
 
-      def initialize(query, title: nil, description: nil, tags: [])
-        super()
+      param :query
+      option :title, default: proc { "OTX search" }
+      option :description, default: proc { "query = #{query}" }
+      option :tags, default: proc { [] }
 
-        @query = query
-        @type = TypeChecker.type(query)
+      attr_reader :type
+
+      def initialize(*args, **kwargs)
+        super
 
-        @title = title || "OTX lookup"
-        @description = description || "query = #{query}"
-        @tags = tags
+        @query = refang(query)
+        @type = TypeChecker.type(query)
       end
 
       def artifacts
-        lookup || []
+        search || []
       end
 
       private
 
-      def config_keys
+      def configuration_keys
         %w[otx_api_key]
       end
 
@@ -40,29 +43,29 @@ module Mihari
         %w[ip domain].include? type
       end
 
-      def lookup
+      def search
         case type
         when "domain"
-          domain_lookup
+          domain_search
         when "ip"
-          ip_lookup
+          ip_search
         else
           raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
         end
       end
 
-      def domain_lookup
+      def domain_search
         records = domain_client.get_passive_dns(query)
-        records.map do |record|
+        records.filter_map do |record|
           record.address if record.record_type == "A"
-        end.compact.uniq
+        end.uniq
       end
 
-      def ip_lookup
+      def ip_search
         records = ip_client.get_passive_dns(query)
-        records.map do |record|
+        records.filter_map do |record|
           record.hostname if record.record_type == "A"
-        end.compact.uniq
+        end.uniq
       end
     end
   end

data/lib/mihari/analyzers/passivetotal.rb

@@ -5,26 +5,29 @@ require "passivetotal"
 module Mihari
   module Analyzers
     class PassiveTotal < Base
-      attr_reader :query, :type, :title, :description, :tags
+      include Mixins::Refang
 
-      def initialize(query, title: nil, description: nil, tags: [])
-        super()
+      param :query
+      option :title, default: proc { "PassiveTotal search" }
+      option :description, default: proc { "query = #{query}" }
+      option :tags, default: proc { [] }
 
-        @query = query
-        @type = TypeChecker.type(query)
+      attr_reader :type
+
+      def initialize(*args, **kwargs)
+        super
 
-        @title = title || "PassiveTotal lookup"
-        @description = description || "query = #{query}"
-        @tags = tags
+        @query = refang(query)
+        @type = TypeChecker.type(query)
       end
 
       def artifacts
-        lookup || []
+        search || []
       end
 
       private
 
-      def config_keys
+      def configuration_keys
         %w[passivetotal_username passivetotal_api_key]
       end
 
@@ -33,30 +36,28 @@ module Mihari
       end
 
       def valid_type?
-        %w[ip domain mail].include? type
+        %w[ip domain mail hash].include? type
       end
 
-      def lookup
+      def search
         case type
-        when "domain"
-          passive_dns_lookup
-        when "ip"
-          passive_dns_lookup
+        when "domain", "ip"
+          passive_dns_search
         when "mail"
-          reverse_whois_lookup
+          reverse_whois_search
         when "hash"
-          ssl_lookup
+          ssl_search
         else
           raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
         end
       end
 
-      def passive_dns_lookup
+      def passive_dns_search
         res = api.dns.passive_unique(query)
         res["results"] || []
       end
 
-      def reverse_whois_lookup
+      def reverse_whois_search
         res = api.whois.search(query: query, field: "email")
         results = res["results"] || []
         results.map do |result|
@@ -64,7 +65,7 @@ module Mihari
         end.flatten.compact.uniq
       end
 
-      def ssl_lookup
+      def ssl_search
         res = api.ssl.history(query)
         results = res["results"] || []
         results.map do |result|

data/lib/mihari/analyzers/pulsedive.rb

@@ -5,26 +5,29 @@ require "pulsedive"
 module Mihari
   module Analyzers
     class Pulsedive < Base
-      attr_reader :query, :type, :title, :description, :tags
+      include Mixins::Refang
 
-      def initialize(query, title: nil, description: nil, tags: [])
-        super()
+      param :query
+      option :title, default: proc { "Pulsedive search" }
+      option :description, default: proc { "query = #{query}" }
+      option :tags, default: proc { [] }
 
-        @query = query
-        @type = TypeChecker.type(query)
+      attr_reader :type
+
+      def initialize(*args, **kwargs)
+        super
 
-        @title = title || "Pulsedive lookup"
-        @description = description || "query = #{query}"
-        @tags = tags
+        @query = refang(query)
+        @type = TypeChecker.type(query)
       end
 
       def artifacts
-        lookup || []
+        search || []
       end
 
       private
 
-      def config_keys
+      def configuration_keys
         %w[pulsedive_api_key]
       end
 
@@ -36,16 +39,16 @@ module Mihari
         %w[ip domain].include? type
       end
 
-      def lookup
+      def search
         raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
 
         indicator = api.indicator.get_by_value(query)
         iid = indicator["iid"]
 
         properties = api.indicator.get_properties_by_id(iid)
-        (properties["dns"] || []).map do |property|
+        (properties["dns"] || []).filter_map do |property|
           property["value"] if ["A", "PTR"].include?(property["name"])
-        end.compact
+        end
       end
     end
   end

data/lib/mihari/analyzers/rule.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+require "uuidtools"
+
+NIL = nil
+
+module Mihari
+  module Analyzers
+    class Rule < Base
+      option :title
+      option :description
+      option :queries
+
+      option :id, default: proc {}
+      option :tags, default: proc { [] }
+      option :allowed_data_types, default: proc { ALLOWED_DATA_TYPES }
+
+      attr_reader :source
+
+      def initialize(**kwargs)
+        super(**kwargs)
+
+        @source = id || UUIDTools::UUID.md5_create(UUIDTools::UUID_URL_NAMESPACE, title + description).to_s
+      end
+
+      ANALYZER_TO_CLASS = {
+        "binaryedge" => BinaryEdge,
+        "censys" => Censys,
+        "circl" => CIRCL,
+        "crtsh" => Crtsh,
+        "dnpedia" => DNPedia,
+        "dnstwister" => DNSTwister,
+        "onyphe" => Onyphe,
+        "otx" => OTX,
+        "passivetotal" => PassiveTotal,
+        "pulsedive" => Pulsedive,
+        "securitytrails" => SecurityTrails,
+        "shodan" => Shodan,
+        "spyse" => Spyse,
+        "urlscan" => Urlscan,
+        "virustotal" => VirusTotal,
+        "zoomeye" => ZoomEye
+      }.freeze
+
+      #
+      # Returns a list of artifacts matched with queries
+      #
+      # @return [Array<Mihari::Artifact>]
+      #
+      def artifacts
+        artifacts = []
+
+        queries.each do |params|
+          analyzer_name = params[:analyzer]
+          klass = get_analyzer_class(analyzer_name)
+
+          query = params[:query]
+          analyzer = klass.new(query, **params)
+
+          # Use #normalized_artifacts method to get atrifacts as Array<Mihari::Artifact>
+          # So Mihari::Artifact object has "source" attribute (e.g. "Shodan")
+          artifacts << analyzer.normalized_artifacts
+        end
+
+        artifacts.flatten
+      end
+
+      #
+      # Normalize artifacts
+      # - Uniquefy artifacts by #uniq(&:data)
+      # - Reject an invalid artifact (for just in case)
+      # - Select artifacts with allowed data types
+      #
+      # @return [Array<Mihari::Artifact>]
+      #
+      def normalized_artifacts
+        @normalized_artifacts ||= artifacts.uniq(&:data).select(&:valid?).select do |artifact|
+          allowed_data_types.include? artifact.data_type
+        end
+      end
+
+      private
+
+      #
+      # Get analyzer class
+      #
+      # @param [String] analyzer_name
+      #
+      # @return [Class<Mihari::Analyzers::Base>] analyzer class
+      #
+      def get_analyzer_class(analyzer_name)
+        analyzer = ANALYZER_TO_CLASS[analyzer_name]
+        return analyzer if analyzer
+
+        raise ArgumentError, "#{analyzer_name} is not supported"
+      end
+    end
+  end
+end