mihari 5.4.2 → 5.4.4

data/lib/mihari/analyzers/rule.rb

@@ -60,9 +60,16 @@ module Mihari
       # @return [Array<Mihari::Artifact>]
       #
       def artifacts
-        analyzers.flat_map(&:normalized_artifacts).map do |artifact|
-          artifact.rule_id = rule.id
-          artifact
+        analyzers.flat_map do |analyzer|
+          result = analyzer.result
+
+          raise result.failure if result.failure? && !analyzer.ignore_error?
+
+          artifacts = result.value!
+          artifacts.map do |artifact|
+            artifact.rule_id = rule.id
+            artifact
+          end
         end
       end
 
@@ -113,11 +120,12 @@ module Mihari
         return [] if enriched_artifacts.empty?
 
         Parallel.map(valid_emitters) do |emitter|
-          emission = emitter.emit
-          Mihari.logger.info "Emission by #{emitter.class} is succeeded"
-          emission
-        rescue StandardError => e
-          Mihari.logger.info "Emission by #{emitter.class} is failed: #{e}"
+          result = emitter.result
+
+          Mihari.logger.info "Emission by #{emitter.class} is failed: #{result.failure}" if result.failure?
+          Mihari.logger.info "Emission by #{emitter.class} is succeeded" if result.success?
+
+          result.value_or nil
         end.compact
       end
 
@@ -127,9 +135,6 @@ module Mihari
       # @return [Mihari::Alert, nil]
       #
       def run
-        # memoize enriched artifacts
-        enriched_artifacts
-
         alert_or_something = bulk_emit
         # returns Mihari::Alert created by the database emitter
         alert_or_something.find { |res| res.is_a?(Mihari::Alert) }
@@ -167,7 +172,7 @@ module Mihari
       # @return [Array<Mihari::Analyzers::Base>]
       #
       def analyzers
-        @analyzers ||= rule.queries.map do |query_params|
+        rule.queries.map do |query_params|
           analyzer_name = query_params[:analyzer]
           klass = get_analyzer_class(analyzer_name)
           klass.from_query(query_params)
@@ -207,7 +212,7 @@ module Mihari
       # @return [Array<Mihari::Emitters::Base>]
       #
       def valid_emitters
-        @valid_emitters ||= emitters.select(&:valid?)
+        emitters.select(&:valid?)
       end
 
       #

data/lib/mihari/analyzers/securitytrails.rb

@@ -30,13 +30,13 @@ module Mihari
       def artifacts
         case type
         when "domain"
-          domain_search
+          client.domain_search query
         when "ip"
-          ip_search
+          client.ip_search query
         when "mail"
-          mail_search
+          client.mail_search query
         else
-          raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
+          raise ValueError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
         end
       end
 
@@ -58,44 +58,6 @@ module Mihari
       def valid_type?
         %w[ip domain mail].include? type
       end
-
-      #
-      # Domain search
-      #
-      # @return [Array<String>]
-      #
-      def domain_search
-        records = client.get_all_dns_history(query, type: "a")
-        records.map do |record|
-          (record["values"] || []).map { |value| value["ip"] }
-        end.flatten.compact.uniq
-      end
-
-      #
-      # IP search
-      #
-      # @return [Array<Mihari::Artifact>]
-      #
-      def ip_search
-        records = client.search_by_ip(query)
-        records.filter_map do |record|
-          data = record["hostname"]
-          Artifact.new(data: data, source: source, metadata: record)
-        end
-      end
-
-      #
-      # Mail search
-      #
-      # @return [Array<String>]
-      #
-      def mail_search
-        records = client.search_by_mail(query)
-        records.filter_map do |record|
-          data = record["hostname"]
-          Artifact.new(data: data, source: source, metadata: record)
-        end
-      end
     end
   end
 end

data/lib/mihari/analyzers/shodan.rb

@@ -18,10 +18,10 @@ module Mihari
       end
 
       def artifacts
-        results = search
-        return [] if results.empty?
-
-        results.map(&:to_artifacts).flatten.uniq(&:data)
+        client.search_with_pagination(
+          query,
+          pagination_limit: pagination_limit
+        ).map(&:artifacts).flatten.uniq(&:data)
       end
 
       def configuration_keys
@@ -30,43 +30,11 @@ module Mihari
 
       private
 
-      PAGE_SIZE = 100
-
-      def client
-        @client ||= Clients::Shodan.new(api_key: api_key)
-      end
-
-      #
-      # Search with pagination
-      #
-      # @param [Integer] page
-      #
-      # @return [Structs::Shodan::Result]
-      #
-      def search_with_page(page: 1)
-        client.search(query, page: page)
-      end
-
       #
-      # Search
+      # @return [Clients::Shodan]
       #
-      # @return [Array<Structs::Shodan::Result>]
-      #
-      def search
-        responses = []
-        (1..pagination_limit).each do |page|
-          res = search_with_page(page: page)
-          responses << res
-          break if res.total <= page * PAGE_SIZE
-
-          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
-          sleep_interval
-        rescue JSON::ParserError
-          # ignore JSON::ParserError
-          # ref. https://github.com/ninoseki/mihari/issues/197
-          next
-        end
-        responses
+      def client
+        @client ||= Clients::Shodan.new(api_key: api_key, interval: interval)
       end
     end
   end

data/lib/mihari/analyzers/urlscan.rb

@@ -4,7 +4,6 @@ module Mihari
   module Analyzers
     class Urlscan < Base
       SUPPORTED_DATA_TYPES = %w[url domain ip].freeze
-      SIZE = 1000
 
       # @return [String, nil]
       attr_reader :api_key
@@ -26,13 +25,12 @@ module Mihari
 
         return if valid_allowed_data_types?
 
-        raise InvalidInputError, "allowed_data_types should be any of url, domain and ip."
+        raise ValueError, "allowed_data_types should be any of url, domain and ip."
       end
 
       def artifacts
-        responses = search
         # @type [Array<Mihari::Artifact>]
-        artifacts = responses.map { |res| res.to_artifacts }.flatten
+        artifacts = client.search_with_pagination(query, pagination_limit: pagination_limit).map(&:artifacts).flatten
 
         artifacts.select do |artifact|
           allowed_data_types.include? artifact.data_type
@@ -46,41 +44,7 @@ module Mihari
       private
 
       def client
-        @client ||= Clients::UrlScan.new(api_key: api_key)
-      end
-
-      #
-      # Search with search_after option
-      #
-      # @return [Structs::Urlscan::Response]
-      #
-      def search_with_search_after(search_after: nil)
-        res = client.search(query, size: SIZE, search_after: search_after)
-        Structs::Urlscan::Response.from_dynamic! res
-      end
-
-      #
-      # Search
-      #
-      # @return [Array<Structs::Urlscan::Response>]
-      #
-      def search
-        responses = []
-
-        search_after = nil
-        pagination_limit.times do
-          res = search_with_search_after(search_after: search_after)
-          responses << res
-
-          break if res.results.length < SIZE
-
-          search_after = res.results.last.sort.join(",")
-
-          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
-          sleep_interval
-        end
-
-        responses
+        @client ||= Clients::UrlScan.new(api_key: api_key, interval: interval)
       end
 
       #

data/lib/mihari/analyzers/virustotal.rb

@@ -31,7 +31,7 @@ module Mihari
         when "ip"
           ip_search
         else
-          raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
+          raise ValueError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
         end
       end
 

data/lib/mihari/analyzers/virustotal_intelligence.rb

@@ -18,7 +18,7 @@ module Mihari
       end
 
       def artifacts
-        search_with_cursor.map(&:to_artifacts).flatten
+        client.intel_search_with_pagination(query, pagination_limit: pagination_limit).map(&:artifacts).flatten
       end
 
       def configuration_keys
@@ -33,30 +33,7 @@ module Mihari
       # @return [::VirusTotal::API]
       #
       def client
-        @client = Clients::VirusTotal.new(api_key: api_key)
-      end
-
-      #
-      # Search with cursor
-      #
-      # @return [Array<Structs::VirusTotalIntelligence::Response>]
-      #
-      def search_with_cursor
-        cursor = nil
-        responses = []
-
-        pagination_limit.times do
-          response = Structs::VirusTotalIntelligence::Response.from_dynamic!(client.intel_search(query,
-            cursor: cursor))
-          responses << response
-          break if response.meta.cursor.nil?
-
-          cursor = response.meta.cursor
-          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
-          sleep_interval
-        end
-
-        responses
+        @client = Clients::VirusTotal.new(api_key: api_key, interval: interval)
       end
     end
   end

data/lib/mihari/analyzers/zoomeye.rb

@@ -25,11 +25,15 @@ module Mihari
       def artifacts
         case type
         when "host"
-          host_search
+          client.host_search_with_pagination(query).map do |res|
+            convert(res)
+          end.flatten
         when "web"
-          web_search
+          client.web_search_with_pagination(query).map do |res|
+            convert(res)
+          end.flatten
         else
-          raise InvalidInputError, "#{type} type is not supported." unless valid_type?
+          raise ValueError, "#{type} type is not supported." unless valid_type?
         end
       end
 
@@ -39,8 +43,6 @@ module Mihari
 
       private
 
-      PAGE_SIZE = 10
-
       #
       # Check whether a type is valid or not
       #
@@ -51,95 +53,27 @@ module Mihari
       end
 
       def client
-        @client ||= Clients::ZoomEye.new(api_key: api_key)
+        @client ||= Clients::ZoomEye.new(api_key: api_key, interval: interval)
       end
 
       #
       # Convert responses into an array of String
       #
-      # @param [Array<Hash>] responses
+      # @param [Hash] response
       #
       # @return [Array<Mihari::Artifact>]
       #
-      def convert_responses(responses)
-        responses.map do |res|
-          matches = res["matches"] || []
-          matches.map do |match|
-            data = match["ip"]
+      def convert(res)
+        matches = res["matches"] || []
+        matches.map do |match|
+          data = match["ip"]
 
-            if data.is_a?(Array)
-              data.map { |d| Artifact.new(data: d, source: source, metadata: match) }
-            else
-              Artifact.new(data: data, source: source, metadata: match)
-            end
+          if data.is_a?(Array)
+            data.map { |d| Artifact.new(data: d, source: source, metadata: match) }
+          else
+            Artifact.new(data: data, source: source, metadata: match)
           end
-        end.flatten.compact.uniq
-      end
-
-      #
-      # Host search
-      #
-      # @param [String] query
-      # @param [Integer] page
-      #
-      # @return [Hash, nil]
-      #
-      def _host_search(query, page: 1)
-        client.host_search(query, page: page)
-      end
-
-      #
-      # Host search
-      #
-      # @return [Array<String>]
-      #
-      def host_search
-        responses = []
-        (1..pagination_limit).each do |page|
-          res = _host_search(query, page: page)
-          break unless res
-
-          total = res["total"].to_i
-          responses << res
-          break if total <= page * PAGE_SIZE
-
-          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
-          sleep_interval
-        end
-        convert_responses responses.compact
-      end
-
-      #
-      # Web search
-      #
-      # @param [String] query
-      # @param [Integer] page
-      #
-      # @return [Hash, nil]
-      #
-      def _web_search(query, page: 1)
-        client.web_search(query, page: page)
-      end
-
-      #
-      # Web search
-      #
-      # @return [Array<String>]
-      #
-      def web_search
-        responses = []
-        (1..pagination_limit).each do |page|
-          res = _web_search(query, page: page)
-          break unless res
-
-          total = res["total"].to_i
-          responses << res
-          break if total <= page * PAGE_SIZE
-
-          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
-          sleep_interval
-        end
-        convert_responses responses.compact
+        end.flatten
       end
     end
   end

data/lib/mihari/clients/base.rb

@@ -9,17 +9,25 @@ module Mihari
       # @return [Hash]
       attr_reader :headers
 
+      # @return [Integer, nil]
+      attr_reader :interval
+
       #
       # @param [String] base_url
       # @param [Hash] headers
       #
-      def initialize(base_url, headers: {})
+      def initialize(base_url, headers: {}, interval: nil)
         @base_url = base_url
         @headers = headers || {}
+        @interval = interval
       end
 
       private
 
+      def sleep_interval
+        sleep(interval) if interval
+      end
+
       #
       # @param [String] path
       #

data/lib/mihari/clients/binaryedge.rb

@@ -7,13 +7,14 @@ module Mihari
       # @param [String] base_url
       # @param [String, nil] api_key
       # @param [Hash] headers
+      # @param [Integer.nil ] interval
       #
-      def initialize(base_url = "https://api.binaryedge.io/v2", api_key:, headers: {})
+      def initialize(base_url = "https://api.binaryedge.io/v2", api_key:, headers: {}, interval: nil)
         raise(ArgumentError, "'api_key' argument is required") unless api_key
 
         headers["x-key"] = api_key
 
-        super(base_url, headers: headers)
+        super(base_url, headers: headers, interval: interval)
       end
 
       #
@@ -21,7 +22,7 @@ module Mihari
       # @param [Integer] page Default 1, Maximum: 500
       # @param [Integer, nil] only_ips If selected, only output IP addresses, ports and protocols.
       #
-      # @return [Hash]
+      # @return [Structs::BinaryEdge::Response]
       #
       def search(query, page: 1, only_ips: nil)
         params = {
@@ -31,7 +32,28 @@ module Mihari
         }.compact
 
         res = get("/query/search", params: params)
-        JSON.parse(res.body.to_s)
+        Structs::BinaryEdge::Response.from_dynamic! JSON.parse(res.body.to_s)
+      end
+
+      #
+      # @param [String] query
+      # @param [Integer, nil] only_ips
+      # @param [Integer] pagination_limit
+      #
+      # @return [Enumerable<Structs::BinaryEdge::Response.>]
+      #
+      def search_with_pagination(query, only_ips: nil, pagination_limit: Mihari.config.pagination_limit)
+        Enumerator.new do |y|
+          (1..pagination_limit).each do |page|
+            res = search(query, page: page, only_ips: only_ips)
+
+            y.yield res
+
+            break if res.events.length < res.pagesize
+
+            sleep_interval
+          end
+        end
       end
     end
   end

data/lib/mihari/clients/censys.rb

@@ -10,14 +10,15 @@ module Mihari
       # @param [String, nil] id
       # @param [String, nil] secret
       # @param [Hash] headers
+      # @param [Integer, nil] interval
       #
-      def initialize(base_url = "https://search.censys.io", id:, secret:, headers: {})
+      def initialize(base_url = "https://search.censys.io", id:, secret:, headers: {}, interval: nil)
         raise(ArgumentError, "'id' argument is required") if id.nil?
         raise(ArgumentError, "'secret' argument is required") if secret.nil?
 
         headers["authorization"] = "Basic #{Base64.strict_encode64("#{id}:#{secret}")}"
 
-        super(base_url, headers: headers)
+        super(base_url, headers: headers, interval: interval)
       end
 
       #
@@ -37,6 +38,35 @@ module Mihari
         res = get("/api/v2/hosts/search", params: params)
         Structs::Censys::Response.from_dynamic! JSON.parse(res.body.to_s)
       end
+
+      #
+      # @param [String] query
+      # @param [Integer, nil] per_page
+      # @param [Integer] pagination_limit
+      #
+      # @return [Enumerable<Structs::Censys::Response>]
+      #
+      def search_with_pagination(query, per_page: nil, pagination_limit: Mihari.config.pagination_limit)
+        cursor = nil
+
+        Enumerator.new do |y|
+          pagination_limit.times do
+            res = search(query, per_page: per_page, cursor: cursor)
+
+            y.yield res
+
+            cursor = res.result.links.next
+            # NOTE: Censys's search API is unstable recently
+            # it may returns empty links or empty string cursors
+            # - Empty links: "links": {}
+            # - Empty cursors: "links": { "next": "", "prev": "" }
+            # So it needs to check both cases
+            break if cursor.nil? || cursor.empty?
+
+            sleep_interval
+          end
+        end
+      end
     end
   end
 end

data/lib/mihari/clients/circl.rb

@@ -20,6 +20,34 @@ module Mihari
         super(base_url, headers: headers)
       end
 
+      #
+      # Passive DNS search
+      #
+      # @param [String] query
+      #
+      # @return [Array<String>]
+      #
+      def passive_dns_search(query)
+        results = dns_query(query)
+        results.filter_map do |result|
+          type = result["rrtype"]
+          (type == "A") ? result["rdata"] : nil
+        end.uniq
+      end
+
+      #
+      # Passive SSL search
+      #
+      # @param [String] query
+      #
+      # @return [Array<String>]
+      #
+      def passive_ssl_search(query)
+        result = ssl_cquery(query)
+        seen = result["seen"] || []
+        seen.uniq
+      end
+
       #
       # @param [String] query
       #
@@ -40,7 +68,6 @@ module Mihari
 
       private
 
-      #
       #
       # @param [String] path
       # @param [Hash] params

data/lib/mihari/clients/crtsh.rb

@@ -18,13 +18,18 @@ module Mihari
       # @param [String, nil] match "=", "ILIKE", "LIKE", "single", "any" or nil
       # @param [String, nil] exclude "expired" or nil
       #
-      # @return [Array<Hash>]
+      # @return [Array<Mihari::Artifact>]
       #
       def search(identity, match: nil, exclude: nil)
         params = { identity: identity, match: match, exclude: exclude, output: "json" }.compact
 
         res = get("/", params: params)
-        JSON.parse(res.body.to_s)
+        parsed = JSON.parse(res.body.to_s)
+
+        parsed.map do |result|
+          values = result["name_value"].to_s.lines.map(&:chomp)
+          values.map { |value| Artifact.new(data: value, metadata: result) }
+        end.flatten
       end
     end
   end

data/lib/mihari/clients/dnstwister.rb

@@ -16,11 +16,13 @@ module Mihari
       #
       # @param [String] domain
       #
-      # @return [Hash]
+      # @return [Array<String>]
       #
       def fuzz(domain)
         res = get("/api/fuzz/#{to_hex(domain)}")
-        JSON.parse(res.body.to_s)
+        res = JSON.parse(res.body.to_s)
+        fuzzy_domains = res["fuzzy_domains"] || []
+        fuzzy_domains.map { |d| d["domain"] }
       end
 
       private