mihari 5.4.2 → 5.4.3

data/lib/mihari/analyzers/zoomeye.rb

@@ -25,9 +25,13 @@ module Mihari
       def artifacts
         case type
         when "host"
-          host_search
+          client.host_search_with_pagination(query).map do |res|
+            convert(res)
+          end.flatten
         when "web"
-          web_search
+          client.web_search_with_pagination(query).map do |res|
+            convert(res)
+          end.flatten
         else
           raise InvalidInputError, "#{type} type is not supported." unless valid_type?
         end
@@ -39,8 +43,6 @@ module Mihari
 
       private
 
-      PAGE_SIZE = 10
-
       #
       # Check whether a type is valid or not
       #
@@ -51,95 +53,27 @@ module Mihari
       end
 
       def client
-        @client ||= Clients::ZoomEye.new(api_key: api_key)
+        @client ||= Clients::ZoomEye.new(api_key: api_key, interval: interval)
       end
 
       #
       # Convert responses into an array of String
       #
-      # @param [Array<Hash>] responses
+      # @param [Hash] response
       #
       # @return [Array<Mihari::Artifact>]
       #
-      def convert_responses(responses)
-        responses.map do |res|
-          matches = res["matches"] || []
-          matches.map do |match|
-            data = match["ip"]
+      def convert(res)
+        matches = res["matches"] || []
+        matches.map do |match|
+          data = match["ip"]
 
-            if data.is_a?(Array)
-              data.map { |d| Artifact.new(data: d, source: source, metadata: match) }
-            else
-              Artifact.new(data: data, source: source, metadata: match)
-            end
+          if data.is_a?(Array)
+            data.map { |d| Artifact.new(data: d, source: source, metadata: match) }
+          else
+            Artifact.new(data: data, source: source, metadata: match)
           end
-        end.flatten.compact.uniq
-      end
-
-      #
-      # Host search
-      #
-      # @param [String] query
-      # @param [Integer] page
-      #
-      # @return [Hash, nil]
-      #
-      def _host_search(query, page: 1)
-        client.host_search(query, page: page)
-      end
-
-      #
-      # Host search
-      #
-      # @return [Array<String>]
-      #
-      def host_search
-        responses = []
-        (1..pagination_limit).each do |page|
-          res = _host_search(query, page: page)
-          break unless res
-
-          total = res["total"].to_i
-          responses << res
-          break if total <= page * PAGE_SIZE
-
-          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
-          sleep_interval
-        end
-        convert_responses responses.compact
-      end
-
-      #
-      # Web search
-      #
-      # @param [String] query
-      # @param [Integer] page
-      #
-      # @return [Hash, nil]
-      #
-      def _web_search(query, page: 1)
-        client.web_search(query, page: page)
-      end
-
-      #
-      # Web search
-      #
-      # @return [Array<String>]
-      #
-      def web_search
-        responses = []
-        (1..pagination_limit).each do |page|
-          res = _web_search(query, page: page)
-          break unless res
-
-          total = res["total"].to_i
-          responses << res
-          break if total <= page * PAGE_SIZE
-
-          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
-          sleep_interval
-        end
-        convert_responses responses.compact
+        end.flatten
       end
     end
   end

data/lib/mihari/clients/base.rb

@@ -9,17 +9,25 @@ module Mihari
       # @return [Hash]
       attr_reader :headers
 
+      # @return [Integer, nil]
+      attr_reader :interval
+
       #
       # @param [String] base_url
       # @param [Hash] headers
       #
-      def initialize(base_url, headers: {})
+      def initialize(base_url, headers: {}, interval: nil)
         @base_url = base_url
         @headers = headers || {}
+        @interval = interval
       end
 
       private
 
+      def sleep_interval
+        sleep(interval) if interval
+      end
+
       #
       # @param [String] path
       #

data/lib/mihari/clients/binaryedge.rb

@@ -7,13 +7,14 @@ module Mihari
       # @param [String] base_url
       # @param [String, nil] api_key
       # @param [Hash] headers
+      # @param [Integer.nil ] interval
       #
-      def initialize(base_url = "https://api.binaryedge.io/v2", api_key:, headers: {})
+      def initialize(base_url = "https://api.binaryedge.io/v2", api_key:, headers: {}, interval: nil)
         raise(ArgumentError, "'api_key' argument is required") unless api_key
 
         headers["x-key"] = api_key
 
-        super(base_url, headers: headers)
+        super(base_url, headers: headers, interval: interval)
       end
 
       #
@@ -33,6 +34,30 @@ module Mihari
         res = get("/query/search", params: params)
         JSON.parse(res.body.to_s)
       end
+
+      #
+      # @param [String] query
+      # @param [Integer, nil] only_ips
+      # @param [Integer] pagination_limit
+      #
+      # @return [Enumerable<Hash>]
+      #
+      def search_with_pagination(query, only_ips: nil, pagination_limit: Mihari.config.pagination_limit)
+        Enumerator.new do |y|
+          (1..pagination_limit).each do |page|
+            res = search(query, page: page, only_ips: only_ips)
+
+            page_size = res["pagesize"].to_i
+            events = res["events"] || []
+
+            y.yield res
+
+            break if events.length < page_size
+
+            sleep_interval
+          end
+        end
+      end
     end
   end
 end

data/lib/mihari/clients/censys.rb

@@ -10,14 +10,15 @@ module Mihari
       # @param [String, nil] id
       # @param [String, nil] secret
       # @param [Hash] headers
+      # @param [Integer, nil] interval
       #
-      def initialize(base_url = "https://search.censys.io", id:, secret:, headers: {})
+      def initialize(base_url = "https://search.censys.io", id:, secret:, headers: {}, interval: nil)
         raise(ArgumentError, "'id' argument is required") if id.nil?
         raise(ArgumentError, "'secret' argument is required") if secret.nil?
 
         headers["authorization"] = "Basic #{Base64.strict_encode64("#{id}:#{secret}")}"
 
-        super(base_url, headers: headers)
+        super(base_url, headers: headers, interval: interval)
       end
 
       #
@@ -37,6 +38,35 @@ module Mihari
         res = get("/api/v2/hosts/search", params: params)
         Structs::Censys::Response.from_dynamic! JSON.parse(res.body.to_s)
       end
+
+      #
+      # @param [String] query
+      # @param [Integer, nil] per_page
+      # @param [Integer] pagination_limit
+      #
+      # @return [Enumerable<Structs::Censys::Response>]
+      #
+      def search_with_pagination(query, per_page: nil, pagination_limit: Mihari.config.pagination_limit)
+        cursor = nil
+
+        Enumerator.new do |y|
+          pagination_limit.times do
+            res = search(query, per_page: per_page, cursor: cursor)
+
+            y.yield res
+
+            cursor = res.result.links.next
+            # NOTE: Censys's search API is unstable recently
+            # it may returns empty links or empty string cursors
+            # - Empty links: "links": {}
+            # - Empty cursors: "links": { "next": "", "prev": "" }
+            # So it needs to check both cases
+            break if cursor.nil? || cursor.empty?
+
+            sleep_interval
+          end
+        end
+      end
     end
   end
 end

data/lib/mihari/clients/circl.rb

@@ -20,6 +20,34 @@ module Mihari
         super(base_url, headers: headers)
       end
 
+      #
+      # Passive DNS search
+      #
+      # @param [String] query
+      #
+      # @return [Array<String>]
+      #
+      def passive_dns_search(query)
+        results = dns_query(query)
+        results.filter_map do |result|
+          type = result["rrtype"]
+          (type == "A") ? result["rdata"] : nil
+        end.uniq
+      end
+
+      #
+      # Passive SSL search
+      #
+      # @param [String] query
+      #
+      # @return [Array<String>]
+      #
+      def passive_ssl_search(query)
+        result = ssl_cquery(query)
+        seen = result["seen"] || []
+        seen.uniq
+      end
+
       #
       # @param [String] query
       #
@@ -40,7 +68,6 @@ module Mihari
 
       private
 
-      #
       #
       # @param [String] path
       # @param [Hash] params

data/lib/mihari/clients/crtsh.rb

@@ -18,13 +18,20 @@ module Mihari
       # @param [String, nil] match "=", "ILIKE", "LIKE", "single", "any" or nil
       # @param [String, nil] exclude "expired" or nil
       #
-      # @return [Array<Hash>]
+      # @return [Array<Mihari::Artifact>]
       #
       def search(identity, match: nil, exclude: nil)
         params = { identity: identity, match: match, exclude: exclude, output: "json" }.compact
 
         res = get("/", params: params)
-        JSON.parse(res.body.to_s)
+        parsed = JSON.parse(res.body.to_s)
+
+        parsed.map do |result|
+          values = result["name_value"].to_s.lines.map(&:chomp)
+          values.map do |value|
+            Artifact.new(data: value, metadata: result)
+          end
+        end.flatten
       end
     end
   end

data/lib/mihari/clients/dnstwister.rb

@@ -16,11 +16,13 @@ module Mihari
       #
       # @param [String] domain
       #
-      # @return [Hash]
+      # @return [Array<String>]
       #
       def fuzz(domain)
         res = get("/api/fuzz/#{to_hex(domain)}")
-        JSON.parse(res.body.to_s)
+        res = JSON.parse(res.body.to_s)
+        fuzzy_domains = res["fuzzy_domains"] || []
+        fuzzy_domains.map { |d| d["domain"] }
       end
 
       private

data/lib/mihari/clients/greynoise.rb

@@ -3,32 +3,59 @@
 module Mihari
   module Clients
     class GreyNoise < Base
+      PAGE_SIZE = 10_000
+
       #
       # @param [String] base_url
       # @param [String, nil] api_key
       # @param [Hash] headers
+      # @param [Integer, nil] interval
       #
-      def initialize(base_url = "https://api.greynoise.io", api_key:, headers: {})
+      def initialize(base_url = "https://api.greynoise.io", api_key:, headers: {}, interval: nil)
         raise(ArgumentError, "'api_key' argument is required") unless api_key
 
         headers["key"] = api_key
-        super(base_url, headers: headers)
+        super(base_url, headers: headers, interval: interval)
       end
 
       #
       # GNQL (GreyNoise Query Language) is a domain-specific query language that uses Lucene deep under the hood
       #
       # @param [String] query GNQL query string
-      # @param [Integer, nil] size Maximum amount of results to grab
+      # @param [Integer] size Maximum amount of results to grab
       # @param [Integer, nil] scroll Scroll token to paginate through results
       #
       # @return [Hash]
       #
-      def gnql_search(query, size: nil, scroll: nil)
+      def gnql_search(query, size: PAGE_SIZE, scroll: nil)
         params = { query: query, size: size, scroll: scroll }.compact
         res = get("/v2/experimental/gnql", params: params)
         Structs::GreyNoise::Response.from_dynamic! JSON.parse(res.body.to_s)
       end
+
+      #
+      # @param [String] query
+      # @param [Integer] size
+      # @param [Integer] pagination_limit
+      #
+      # @return [Enumerable<Structs::GreyNoise::Response>]
+      #
+      def gnql_search_with_pagination(query, size: PAGE_SIZE, pagination_limit: Mihari.config.pagination_limit)
+        scroll = nil
+
+        Enumerator.new do |y|
+          pagination_limit.times do
+            res = gnql_search(query, size: size, scroll: scroll)
+
+            y.yield res
+
+            scroll = res.scroll
+            break if scroll.nil?
+
+            sleep_interval
+          end
+        end
+      end
     end
   end
 end

data/lib/mihari/clients/hunterhow.rb

@@ -5,6 +5,8 @@ require "base64"
 module Mihari
   module Clients
     class HunterHow < Base
+      PAGE_SIZE = 100
+
       # @return [String]
       attr_reader :api_key
 
@@ -12,11 +14,12 @@ module Mihari
       # @param [String] base_url
       # @param [String, nil] api_key
       # @param [Hash] headers
+      # @param [Integer, nil] interval
       #
-      def initialize(base_url = "https://api.hunter.how/", api_key:, headers: {})
+      def initialize(base_url = "https://api.hunter.how/", api_key:, headers: {}, interval: nil)
         raise(ArgumentError, "'api_key' argument is required") unless api_key
 
-        super(base_url, headers: headers)
+        super(base_url, headers: headers, interval: interval)
 
         @api_key = api_key
       end
@@ -30,7 +33,7 @@ module Mihari
       #
       # @return [Structs::HunterHow::Response]
       #
-      def search(query, start_time:, end_time:, page: 1, page_size: 10)
+      def search(query, start_time:, end_time:, page: 1, page_size: PAGE_SIZE)
         params = {
           query: Base64.urlsafe_encode64(query),
           page: page,
@@ -42,6 +45,41 @@ module Mihari
         res = get("/search", params: params)
         Structs::HunterHow::Response.from_dynamic! JSON.parse(res.body.to_s)
       end
+
+      #
+      # @param [String] query String used to query our data
+      # @param [Integer] page_size Default 100, Maximum: 100
+      # @param [Integer] pagination_limit
+      # @param [String] start_time
+      # @param [String] end_time
+      #
+      # @return [Enumerable<Structs::HunterHow::Response>]
+      #
+      def search_with_pagination(
+        query,
+        start_time:,
+        end_time:,
+        page_size: PAGE_SIZE,
+        pagination_limit: Mihari.config.pagination_limit
+      )
+        Enumerator.new do |y|
+          (1..pagination_limit).each do |page|
+            res = search(
+              query,
+              start_time: start_time,
+              end_time: end_time,
+              page: page,
+              page_size: page_size
+            )
+
+            y.yield res
+
+            break if res.data.list.length < page_size
+
+            sleep_interval
+          end
+        end
+      end
     end
   end
 end

data/lib/mihari/clients/onyphe.rb

@@ -3,6 +3,8 @@
 module Mihari
   module Clients
     class Onyphe < Base
+      PAGE_SIZE = 10
+
       # @return [String]
       attr_reader :api_key
 
@@ -11,10 +13,10 @@ module Mihari
       # @param [String, nil] api_key
       # @param [Hash] headers
       #
-      def initialize(base_url = "https://www.onyphe.io", api_key:, headers: {})
+      def initialize(base_url = "https://www.onyphe.io", api_key:, headers: {}, interval: nil)
         raise(ArgumentError, "'api_key' argument is required") if api_key.nil?
 
-        super(base_url, headers: headers)
+        super(base_url, headers: headers, interval: interval)
 
         @api_key = api_key
       end
@@ -23,13 +25,33 @@ module Mihari
       # @param [String] query
       # @param [Integer] page
       #
-      # @return [Hash]
+      # @return [Structs::Onyphe::Response]
       #
       def datascan(query, page: 1)
         params = { page: page, apikey: api_key }
         res = get("/api/v2/simple/datascan/#{query}", params: params)
         Structs::Onyphe::Response.from_dynamic! JSON.parse(res.body.to_s)
       end
+
+      #
+      # @param [String] query
+      # @param [Integer] pagination_limit
+      #
+      # @return [Enumerable<Structs::Onyphe::Response>]
+      #
+      def datascan_with_pagination(query, pagination_limit: Mihari.config.pagination_limit)
+        Enumerator.new do |y|
+          (1..pagination_limit).each do |page|
+            res = datascan(query, page: page)
+
+            y.yield res
+
+            break if res.total <= page * PAGE_SIZE
+
+            sleep_interval
+          end
+        end
+      end
     end
   end
 end

data/lib/mihari/clients/otx.rb

@@ -15,6 +15,46 @@ module Mihari
         super(base_url, headers: headers)
       end
 
+      #
+      # Domain search
+      #
+      # @param [String] query
+      #
+      # @return [Array<String>]
+      #
+      def domain_search(query)
+        res = query_by_domain(query)
+        return [] if res.nil?
+
+        records = res["passive_dns"] || []
+        records.filter_map do |record|
+          record_type = record["record_type"]
+          address = record["address"]
+
+          address if record_type == "A"
+        end.uniq
+      end
+
+      #
+      # IP search
+      #
+      # @param [String] query
+      #
+      # @return [Array<String>]
+      #
+      def ip_search(query)
+        res = query_by_ip(query)
+        return [] if res.nil?
+
+        records = res["passive_dns"] || []
+        records.filter_map do |record|
+          record_type = record["record_type"]
+          hostname = record["hostname"]
+
+          hostname if record_type == "A"
+        end.uniq
+      end
+
       #
       # @param [String] ip
       #

data/lib/mihari/clients/passivetotal.rb

@@ -21,35 +21,53 @@ module Mihari
       end
 
       #
-      # @param [String] query
-      #
-      def ssl_search(query)
-        params = { query: query }
-        _get("/v2/ssl-certificate/history", params: params)
-      end
-
+      # Passive DNS search
       #
       # @param [String] query
       #
-      # @return [Hash]
+      # @return [Array<String>]
       #
       def passive_dns_search(query)
         params = { query: query }
-        _get("/v2/dns/passive/unique", params: params)
+        res = _get("/v2/dns/passive/unique", params: params)
+        res["results"] || []
       end
 
       #
-      # @param [String] query the domain being queried
-      # @param [String] field whether to return historical results
+      # Reverse whois search
       #
-      # @return [Hash]
+      # @param [String] query
+      #
+      # @return [Array<Mihari::Artifact>]
       #
-      def reverse_whois_search(query:, field:)
+      def reverse_whois_search(query)
         params = {
           query: query,
-          field: field
+          field: "email"
         }.compact
-        _get("/v2/whois/search", params: params)
+        res = _get("/v2/whois/search", params: params)
+        results = res["results"] || []
+        results.map do |result|
+          data = result["domain"]
+          Artifact.new(data: data, metadata: result)
+        end.flatten
+      end
+
+      #
+      # Passive SSL search
+      #
+      # @param [String] query
+      #
+      # @return [Array<Mihari::Artifact>]
+      #
+      def ssl_search(query)
+        params = { query: query }
+        res = _get("/v2/ssl-certificate/history", params: params)
+        results = res["results"] || []
+        results.map do |result|
+          data = result["ipAddresses"]
+          data.map { |d| Artifact.new(data: d, metadata: result) }
+        end.flatten
       end
 
       private