mihari 5.4.1 → 5.4.3

data/lib/mihari/analyzers/base.rb

@@ -82,10 +82,6 @@ module Mihari
 
       private
 
-      def sleep_interval
-        sleep(interval) if interval
-      end
-
       class << self
         #
         # Initialize an analyzer by query params

data/lib/mihari/analyzers/binaryedge.rb

@@ -18,16 +18,13 @@ module Mihari
       end
 
       def artifacts
-        results = search
-        return [] unless results || results.empty?
-
-        results.map do |result|
-          events = result["events"] || []
+        client.search_with_pagination(query, pagination_limit: pagination_limit).map do |res|
+          events = res["events"] || []
           events.filter_map do |event|
             data = event.dig("target", "ip")
             data.nil? ? nil : Artifact.new(data: data, source: source, metadata: event)
           end
-        end.flatten
+        end
       end
 
       def configuration_keys
@@ -36,49 +33,12 @@ module Mihari
 
       private
 
-      PAGE_SIZE = 20
-
-      #
-      # Search with pagination
-      #
-      # @param [Integer] page
-      #
-      # @return [Hash]
-      #
-      def search_with_page(page: 1)
-        client.search(query, page: page)
-      rescue StatusCodeError => e
-        raise RetryableError, e if e.message.include?("Request time limit exceeded")
-
-        raise e
-      end
-
-      #
-      # Search
-      #
-      # @return [Array<Hash>]
-      #
-      def search
-        responses = []
-        (1..pagination_limit).each do |page|
-          res = search_with_page(page: page)
-          total = res["total"].to_i
-
-          responses << res
-          break if total <= page * PAGE_SIZE
-
-          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
-          sleep_interval
-        end
-        responses
-      end
-
       #
       #
       # @return [Mihari::Clients::BinaryEdge]
       #
       def client
-        @client ||= Clients::BinaryEdge.new(api_key: api_key)
+        @client ||= Clients::BinaryEdge.new(api_key: api_key, interval: interval)
       end
     end
   end

data/lib/mihari/analyzers/censys.rb

@@ -27,25 +27,9 @@ module Mihari
       # @return [Array<Mihari::Artifact>]
       #
       def artifacts
-        artifacts = []
-
-        cursor = nil
-        pagination_limit.times do
-          response = client.search(query, cursor: cursor)
-          artifacts << response.result.to_artifacts
-          cursor = response.result.links.next
-          # NOTE: Censys's search API is unstable recently
-          # it may returns empty links or empty string cursors
-          # - Empty links: "links": {}
-          # - Empty cursors: "links": { "next": "", "prev": "" }
-          # So it needs to check both cases
-          break if cursor.nil? || cursor.empty?
-
-          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
-          sleep_interval
-        end
-
-        artifacts.flatten.uniq(&:data)
+        client.search_with_pagination(query, pagination_limit: pagination_limit).map do |res|
+          res.result.artifacts
+        end.flatten.uniq(&:data)
       end
 
       #
@@ -68,7 +52,7 @@ module Mihari
       # @return [Mihari::Clients::Censys]
       #
       def client
-        @client ||= Clients::Censys.new(id: id, secret: secret)
+        @client ||= Clients::Censys.new(id: id, secret: secret, interval: interval)
       end
 
       #

data/lib/mihari/analyzers/circl.rb

@@ -32,9 +32,9 @@ module Mihari
       def artifacts
         case type
         when "domain"
-          passive_dns_search
+          client.passive_dns_search query
         when "hash"
-          passive_ssl_search
+          client.passive_ssl_search query
         else
           raise InvalidInputError, "#{@query}(type: #{@type || "unknown"}) is not supported."
         end
@@ -54,30 +54,6 @@ module Mihari
         @client ||= Clients::CIRCL.new(username: username, password: password)
       end
 
-      #
-      # Passive DNS search
-      #
-      # @return [Array<String>]
-      #
-      def passive_dns_search
-        results = client.dns_query(query)
-        results.filter_map do |result|
-          type = result["rrtype"]
-          (type == "A") ? result["rdata"] : nil
-        end.uniq
-      end
-
-      #
-      # Passive SSL search
-      #
-      # @return [Array<String>]
-      #
-      def passive_ssl_search
-        result = client.ssl_cquery(query)
-        seen = result["seen"] || []
-        seen.uniq
-      end
-
       def username?
         !username.nil?
       end

data/lib/mihari/analyzers/crtsh.rb

@@ -18,13 +18,8 @@ module Mihari
       end
 
       def artifacts
-        results = search
-        results.map do |result|
-          values = result["name_value"].to_s.lines.map(&:chomp)
-          values.map do |value|
-            Artifact.new(data: value, source: source, metadata: result)
-          end
-        end.flatten
+        exclude = exclude_expired ? "expired" : nil
+        client.search(query, exclude: exclude)
       end
 
       private
@@ -35,16 +30,6 @@ module Mihari
       def client
         @client ||= Mihari::Clients::Crtsh.new
       end
-
-      #
-      # Search
-      #
-      # @return [Array<Hash>]
-      #
-      def search
-        exclude = exclude_expired ? "expired" : nil
-        client.search(query, exclude: exclude)
-      end
     end
   end
 end

data/lib/mihari/analyzers/dnstwister.rb

@@ -21,9 +21,7 @@ module Mihari
       def artifacts
         raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
 
-        res = client.fuzz(query)
-        fuzzy_domains = res["fuzzy_domains"] || []
-        domains = fuzzy_domains.map { |domain| domain["domain"] }
+        domains = client.fuzz(query)
         Parallel.map(domains) do |domain|
           resolvable?(domain) ? domain : nil
         end.compact

data/lib/mihari/analyzers/greynoise.rb

@@ -3,8 +3,6 @@
 module Mihari
   module Analyzers
     class GreyNoise < Base
-      PAGE_SIZE = 10_000
-
       # @return [String, nil]
       attr_reader :api_key
 
@@ -20,7 +18,10 @@ module Mihari
       end
 
       def artifacts
-        client.gnql_search(query, size: PAGE_SIZE).to_artifacts
+        client.gnql_search_with_pagination(
+          query,
+          pagination_limit: pagination_limit
+        ).map(&:artifacts).flatten
       end
 
       def configuration_keys
@@ -30,7 +31,7 @@ module Mihari
       private
 
       def client
-        @client ||= Clients::GreyNoise.new(api_key: api_key)
+        @client ||= Clients::GreyNoise.new(api_key: api_key, interval: interval)
       end
     end
   end

data/lib/mihari/analyzers/hunterhow.rb

@@ -30,25 +30,13 @@ module Mihari
       # @return [Array<Mihari::Artifact>]
       #
       def artifacts
-        artifacts = []
-
-        (1..pagination_limit).each do |page|
-          res = client.search(
-            query,
-            page: page,
-            page_size: PAGE_SIZE,
-            start_time: start_time.strftime("%Y-%m-%d"),
-            end_time: end_time.strftime("%Y-%m-%d")
-          )
-
-          artifacts << res.data.artifacts
-
-          break if res.data.list.length < PAGE_SIZE
-
-          sleep_interval
-        end
-
-        artifacts.flatten
+        client.search_with_pagination(
+          query,
+          start_time: start_time.strftime("%Y-%m-%d"),
+          end_time: end_time.strftime("%Y-%m-%d")
+        ).map do |res|
+          res.data.artifacts
+        end.flatten
       end
 
       def configuration_keys
@@ -57,11 +45,8 @@ module Mihari
 
       private
 
-      # @return [Integer]
-      PAGE_SIZE = 100
-
       def client
-        @client ||= Clients::HunterHow.new(api_key: api_key)
+        @client ||= Clients::HunterHow.new(api_key: api_key, interval: interval)
       end
     end
   end

data/lib/mihari/analyzers/onyphe.rb

@@ -20,10 +20,10 @@ module Mihari
       end
 
       def artifacts
-        responses = search
-        return [] unless responses
-
-        responses.map(&:to_artifacts).flatten
+        client.datascan_with_pagination(
+          query,
+          pagination_limit: pagination_limit
+        ).map(&:artifacts).flatten
       end
 
       def configuration_keys
@@ -32,42 +32,8 @@ module Mihari
 
       private
 
-      PAGE_SIZE = 10
-
       def client
-        @client ||= Clients::Onyphe.new(api_key: api_key)
-      end
-
-      #
-      # Search with pagination
-      #
-      # @param [String] query
-      # @param [Integer] page
-      #
-      # @return [Structs::Onyphe::Response]
-      #
-      def search_with_page(query, page: 1)
-        client.datascan(query, page: page)
-      end
-
-      #
-      # Search
-      #
-      # @return [Array<Structs::Onyphe::Response>]
-      #
-      def search
-        responses = []
-        (1..pagination_limit).each do |page|
-          res = search_with_page(query, page: page)
-          responses << res
-
-          total = res.total
-          break if total <= page * PAGE_SIZE
-
-          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
-          sleep_interval
-        end
-        responses
+        @client ||= Clients::Onyphe.new(api_key: api_key, interval: interval)
       end
     end
   end

data/lib/mihari/analyzers/otx.rb

@@ -27,9 +27,9 @@ module Mihari
       def artifacts
         case type
         when "domain"
-          domain_search
+          client.domain_search(query)
         when "ip"
-          ip_search
+          client.ip_search(query)
         else
           raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
         end
@@ -53,42 +53,6 @@ module Mihari
       def valid_type?
         %w[ip domain].include? type
       end
-
-      #
-      # Domain search
-      #
-      # @return [Array<String>]
-      #
-      def domain_search
-        res = client.query_by_domain(query)
-        return [] if res.nil?
-
-        records = res["passive_dns"] || []
-        records.filter_map do |record|
-          record_type = record["record_type"]
-          address = record["address"]
-
-          address if record_type == "A"
-        end.uniq
-      end
-
-      #
-      # IP search
-      #
-      # @return [Array<String>]
-      #
-      def ip_search
-        res = client.query_by_ip(query)
-        return [] if res.nil?
-
-        records = res["passive_dns"] || []
-        records.filter_map do |record|
-          record_type = record["record_type"]
-          hostname = record["hostname"]
-
-          hostname if record_type == "A"
-        end.uniq
-      end
     end
   end
 end

data/lib/mihari/analyzers/passivetotal.rb

@@ -32,11 +32,11 @@ module Mihari
       def artifacts
         case type
         when "domain", "ip"
-          passive_dns_search
+          client.passive_dns_search query
         when "mail"
-          reverse_whois_search
+          client.reverse_whois_search query
         when "hash"
-          ssl_search
+          client.ssl_search query
         else
           raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
         end
@@ -65,44 +65,6 @@ module Mihari
         %w[ip domain mail hash].include? type
       end
 
-      #
-      # Passive DNS search
-      #
-      # @return [Array<String>]
-      #
-      def passive_dns_search
-        res = client.passive_dns_search(query)
-        res["results"] || []
-      end
-
-      #
-      # Reverse whois search
-      #
-      # @return [Array<Mihari::Artifact>]
-      #
-      def reverse_whois_search
-        res = client.reverse_whois_search(query: query, field: "email")
-        results = res["results"] || []
-        results.map do |result|
-          data = result["domain"]
-          Artifact.new(data: data, source: source, metadata: result)
-        end.flatten
-      end
-
-      #
-      # Passive SSL search
-      #
-      # @return [Array<Mihari::Artifact>]
-      #
-      def ssl_search
-        res = client.ssl_search(query)
-        results = res["results"] || []
-        results.map do |result|
-          data = result["ipAddresses"]
-          data.map { |d| Artifact.new(data: d, source: source, metadata: result) }
-        end.flatten
-      end
-
       def username?
         !username.nil?
       end

data/lib/mihari/analyzers/securitytrails.rb

@@ -30,11 +30,11 @@ module Mihari
       def artifacts
         case type
         when "domain"
-          domain_search
+          client.domain_search query
         when "ip"
-          ip_search
+          client.ip_search query
         when "mail"
-          mail_search
+          client.mail_search query
         else
           raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
         end
@@ -58,44 +58,6 @@ module Mihari
       def valid_type?
         %w[ip domain mail].include? type
       end
-
-      #
-      # Domain search
-      #
-      # @return [Array<String>]
-      #
-      def domain_search
-        records = client.get_all_dns_history(query, type: "a")
-        records.map do |record|
-          (record["values"] || []).map { |value| value["ip"] }
-        end.flatten.compact.uniq
-      end
-
-      #
-      # IP search
-      #
-      # @return [Array<Mihari::Artifact>]
-      #
-      def ip_search
-        records = client.search_by_ip(query)
-        records.filter_map do |record|
-          data = record["hostname"]
-          Artifact.new(data: data, source: source, metadata: record)
-        end
-      end
-
-      #
-      # Mail search
-      #
-      # @return [Array<String>]
-      #
-      def mail_search
-        records = client.search_by_mail(query)
-        records.filter_map do |record|
-          data = record["hostname"]
-          Artifact.new(data: data, source: source, metadata: record)
-        end
-      end
     end
   end
 end

data/lib/mihari/analyzers/shodan.rb

@@ -18,10 +18,10 @@ module Mihari
       end
 
       def artifacts
-        results = search
-        return [] if results.empty?
-
-        results.map(&:to_artifacts).flatten.uniq(&:data)
+        client.search_with_pagination(
+          query,
+          pagination_limit: pagination_limit
+        ).map(&:artifacts).flatten.uniq(&:data)
       end
 
       def configuration_keys
@@ -30,43 +30,11 @@ module Mihari
 
       private
 
-      PAGE_SIZE = 100
-
-      def client
-        @client ||= Clients::Shodan.new(api_key: api_key)
-      end
-
-      #
-      # Search with pagination
-      #
-      # @param [Integer] page
-      #
-      # @return [Structs::Shodan::Result]
-      #
-      def search_with_page(page: 1)
-        client.search(query, page: page)
-      end
-
       #
-      # Search
+      # @return [Clients::Shodan]
       #
-      # @return [Array<Structs::Shodan::Result>]
-      #
-      def search
-        responses = []
-        (1..pagination_limit).each do |page|
-          res = search_with_page(page: page)
-          responses << res
-          break if res.total <= page * PAGE_SIZE
-
-          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
-          sleep_interval
-        rescue JSON::ParserError
-          # ignore JSON::ParserError
-          # ref. https://github.com/ninoseki/mihari/issues/197
-          next
-        end
-        responses
+      def client
+        @client ||= Clients::Shodan.new(api_key: api_key, interval: interval)
       end
     end
   end

data/lib/mihari/analyzers/urlscan.rb

@@ -4,7 +4,6 @@ module Mihari
   module Analyzers
     class Urlscan < Base
       SUPPORTED_DATA_TYPES = %w[url domain ip].freeze
-      SIZE = 1000
 
       # @return [String, nil]
       attr_reader :api_key
@@ -30,9 +29,8 @@ module Mihari
       end
 
       def artifacts
-        responses = search
         # @type [Array<Mihari::Artifact>]
-        artifacts = responses.map { |res| res.to_artifacts }.flatten
+        artifacts = client.search_with_pagination(query, pagination_limit: pagination_limit).map(&:artifacts).flatten
 
         artifacts.select do |artifact|
           allowed_data_types.include? artifact.data_type
@@ -46,41 +44,7 @@ module Mihari
       private
 
       def client
-        @client ||= Clients::UrlScan.new(api_key: api_key)
-      end
-
-      #
-      # Search with search_after option
-      #
-      # @return [Structs::Urlscan::Response]
-      #
-      def search_with_search_after(search_after: nil)
-        res = client.search(query, size: SIZE, search_after: search_after)
-        Structs::Urlscan::Response.from_dynamic! res
-      end
-
-      #
-      # Search
-      #
-      # @return [Array<Structs::Urlscan::Response>]
-      #
-      def search
-        responses = []
-
-        search_after = nil
-        pagination_limit.times do
-          res = search_with_search_after(search_after: search_after)
-          responses << res
-
-          break if res.results.length < SIZE
-
-          search_after = res.results.last.sort.join(",")
-
-          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
-          sleep_interval
-        end
-
-        responses
+        @client ||= Clients::UrlScan.new(api_key: api_key, interval: interval)
       end
 
       #

data/lib/mihari/analyzers/virustotal_intelligence.rb

@@ -18,7 +18,7 @@ module Mihari
       end
 
       def artifacts
-        search_with_cursor.map(&:to_artifacts).flatten
+        client.intel_search_with_pagination(query, pagination_limit: pagination_limit).map(&:artifacts).flatten
       end
 
       def configuration_keys
@@ -33,30 +33,7 @@ module Mihari
       # @return [::VirusTotal::API]
       #
       def client
-        @client = Clients::VirusTotal.new(api_key: api_key)
-      end
-
-      #
-      # Search with cursor
-      #
-      # @return [Array<Structs::VirusTotalIntelligence::Response>]
-      #
-      def search_with_cursor
-        cursor = nil
-        responses = []
-
-        pagination_limit.times do
-          response = Structs::VirusTotalIntelligence::Response.from_dynamic!(client.intel_search(query,
-            cursor: cursor))
-          responses << response
-          break if response.meta.cursor.nil?
-
-          cursor = response.meta.cursor
-          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
-          sleep_interval
-        end
-
-        responses
+        @client = Clients::VirusTotal.new(api_key: api_key, interval: interval)
       end
     end
   end