mihari 5.4.2 → 5.4.4

data/lib/mihari/clients/greynoise.rb

@@ -3,32 +3,59 @@
 module Mihari
   module Clients
     class GreyNoise < Base
+      PAGE_SIZE = 10_000
+
       #
       # @param [String] base_url
       # @param [String, nil] api_key
       # @param [Hash] headers
+      # @param [Integer, nil] interval
       #
-      def initialize(base_url = "https://api.greynoise.io", api_key:, headers: {})
+      def initialize(base_url = "https://api.greynoise.io", api_key:, headers: {}, interval: nil)
         raise(ArgumentError, "'api_key' argument is required") unless api_key
 
         headers["key"] = api_key
-        super(base_url, headers: headers)
+        super(base_url, headers: headers, interval: interval)
       end
 
       #
       # GNQL (GreyNoise Query Language) is a domain-specific query language that uses Lucene deep under the hood
       #
       # @param [String] query GNQL query string
-      # @param [Integer, nil] size Maximum amount of results to grab
+      # @param [Integer] size Maximum amount of results to grab
       # @param [Integer, nil] scroll Scroll token to paginate through results
       #
       # @return [Hash]
       #
-      def gnql_search(query, size: nil, scroll: nil)
+      def gnql_search(query, size: PAGE_SIZE, scroll: nil)
         params = { query: query, size: size, scroll: scroll }.compact
         res = get("/v2/experimental/gnql", params: params)
         Structs::GreyNoise::Response.from_dynamic! JSON.parse(res.body.to_s)
       end
+
+      #
+      # @param [String] query
+      # @param [Integer] size
+      # @param [Integer] pagination_limit
+      #
+      # @return [Enumerable<Structs::GreyNoise::Response>]
+      #
+      def gnql_search_with_pagination(query, size: PAGE_SIZE, pagination_limit: Mihari.config.pagination_limit)
+        scroll = nil
+
+        Enumerator.new do |y|
+          pagination_limit.times do
+            res = gnql_search(query, size: size, scroll: scroll)
+
+            y.yield res
+
+            scroll = res.scroll
+            break if scroll.nil?
+
+            sleep_interval
+          end
+        end
+      end
     end
   end
 end

data/lib/mihari/clients/hunterhow.rb

@@ -5,6 +5,8 @@ require "base64"
 module Mihari
   module Clients
     class HunterHow < Base
+      PAGE_SIZE = 100
+
       # @return [String]
       attr_reader :api_key
 
@@ -12,11 +14,12 @@ module Mihari
       # @param [String] base_url
       # @param [String, nil] api_key
       # @param [Hash] headers
+      # @param [Integer, nil] interval
       #
-      def initialize(base_url = "https://api.hunter.how/", api_key:, headers: {})
+      def initialize(base_url = "https://api.hunter.how/", api_key:, headers: {}, interval: nil)
         raise(ArgumentError, "'api_key' argument is required") unless api_key
 
-        super(base_url, headers: headers)
+        super(base_url, headers: headers, interval: interval)
 
         @api_key = api_key
       end
@@ -30,7 +33,7 @@ module Mihari
       #
       # @return [Structs::HunterHow::Response]
       #
-      def search(query, start_time:, end_time:, page: 1, page_size: 10)
+      def search(query, start_time:, end_time:, page: 1, page_size: PAGE_SIZE)
         params = {
           query: Base64.urlsafe_encode64(query),
           page: page,
@@ -42,6 +45,41 @@ module Mihari
         res = get("/search", params: params)
         Structs::HunterHow::Response.from_dynamic! JSON.parse(res.body.to_s)
       end
+
+      #
+      # @param [String] query String used to query our data
+      # @param [Integer] page_size Default 100, Maximum: 100
+      # @param [Integer] pagination_limit
+      # @param [String] start_time
+      # @param [String] end_time
+      #
+      # @return [Enumerable<Structs::HunterHow::Response>]
+      #
+      def search_with_pagination(
+        query,
+        start_time:,
+        end_time:,
+        page_size: PAGE_SIZE,
+        pagination_limit: Mihari.config.pagination_limit
+      )
+        Enumerator.new do |y|
+          (1..pagination_limit).each do |page|
+            res = search(
+              query,
+              start_time: start_time,
+              end_time: end_time,
+              page: page,
+              page_size: page_size
+            )
+
+            y.yield res
+
+            break if res.data.list.length < page_size
+
+            sleep_interval
+          end
+        end
+      end
     end
   end
 end

data/lib/mihari/clients/onyphe.rb

@@ -3,6 +3,8 @@
 module Mihari
   module Clients
     class Onyphe < Base
+      PAGE_SIZE = 10
+
       # @return [String]
       attr_reader :api_key
 
@@ -11,10 +13,10 @@ module Mihari
       # @param [String, nil] api_key
       # @param [Hash] headers
       #
-      def initialize(base_url = "https://www.onyphe.io", api_key:, headers: {})
+      def initialize(base_url = "https://www.onyphe.io", api_key:, headers: {}, interval: nil)
         raise(ArgumentError, "'api_key' argument is required") if api_key.nil?
 
-        super(base_url, headers: headers)
+        super(base_url, headers: headers, interval: interval)
 
         @api_key = api_key
       end
@@ -23,13 +25,33 @@ module Mihari
       # @param [String] query
       # @param [Integer] page
       #
-      # @return [Hash]
+      # @return [Structs::Onyphe::Response]
       #
       def datascan(query, page: 1)
         params = { page: page, apikey: api_key }
         res = get("/api/v2/simple/datascan/#{query}", params: params)
         Structs::Onyphe::Response.from_dynamic! JSON.parse(res.body.to_s)
       end
+
+      #
+      # @param [String] query
+      # @param [Integer] pagination_limit
+      #
+      # @return [Enumerable<Structs::Onyphe::Response>]
+      #
+      def datascan_with_pagination(query, pagination_limit: Mihari.config.pagination_limit)
+        Enumerator.new do |y|
+          (1..pagination_limit).each do |page|
+            res = datascan(query, page: page)
+
+            y.yield res
+
+            break if res.total <= page * PAGE_SIZE
+
+            sleep_interval
+          end
+        end
+      end
     end
   end
 end

data/lib/mihari/clients/otx.rb

@@ -15,6 +15,46 @@ module Mihari
         super(base_url, headers: headers)
       end
 
+      #
+      # Domain search
+      #
+      # @param [String] query
+      #
+      # @return [Array<String>]
+      #
+      def domain_search(query)
+        res = query_by_domain(query)
+        return [] if res.nil?
+
+        records = res["passive_dns"] || []
+        records.filter_map do |record|
+          record_type = record["record_type"]
+          address = record["address"]
+
+          address if record_type == "A"
+        end.uniq
+      end
+
+      #
+      # IP search
+      #
+      # @param [String] query
+      #
+      # @return [Array<String>]
+      #
+      def ip_search(query)
+        res = query_by_ip(query)
+        return [] if res.nil?
+
+        records = res["passive_dns"] || []
+        records.filter_map do |record|
+          record_type = record["record_type"]
+          hostname = record["hostname"]
+
+          hostname if record_type == "A"
+        end.uniq
+      end
+
       #
       # @param [String] ip
       #

data/lib/mihari/clients/passivetotal.rb

@@ -21,35 +21,53 @@ module Mihari
       end
 
       #
-      # @param [String] query
-      #
-      def ssl_search(query)
-        params = { query: query }
-        _get("/v2/ssl-certificate/history", params: params)
-      end
-
+      # Passive DNS search
       #
       # @param [String] query
       #
-      # @return [Hash]
+      # @return [Array<String>]
       #
       def passive_dns_search(query)
         params = { query: query }
-        _get("/v2/dns/passive/unique", params: params)
+        res = _get("/v2/dns/passive/unique", params: params)
+        res["results"] || []
       end
 
       #
-      # @param [String] query the domain being queried
-      # @param [String] field whether to return historical results
+      # Reverse whois search
       #
-      # @return [Hash]
+      # @param [String] query
+      #
+      # @return [Array<Mihari::Artifact>]
       #
-      def reverse_whois_search(query:, field:)
+      def reverse_whois_search(query)
         params = {
           query: query,
-          field: field
+          field: "email"
         }.compact
-        _get("/v2/whois/search", params: params)
+        res = _get("/v2/whois/search", params: params)
+        results = res["results"] || []
+        results.map do |result|
+          data = result["domain"]
+          Artifact.new(data: data, metadata: result)
+        end.flatten
+      end
+
+      #
+      # Passive SSL search
+      #
+      # @param [String] query
+      #
+      # @return [Array<Mihari::Artifact>]
+      #
+      def ssl_search(query)
+        params = { query: query }
+        res = _get("/v2/ssl-certificate/history", params: params)
+        results = res["results"] || []
+        results.map do |result|
+          data = result["ipAddresses"]
+          data.map { |d| Artifact.new(data: d, metadata: result) }
+        end.flatten
       end
 
       private

data/lib/mihari/clients/publsedive.rb

@@ -49,7 +49,7 @@ module Mihari
         params["key"] = api_key
 
         res = get(path, params: params)
-        JSON.parse(res.body.to_s)
+        JSON.parse res.body.to_s
       end
     end
   end

data/lib/mihari/clients/securitytrails.rb

@@ -15,6 +15,50 @@ module Mihari
         super(base_url, headers: headers)
       end
 
+      #
+      # Domain search
+      #
+      # @param [String] query
+      #
+      # @return [Array<String>]
+      #
+      def domain_search(query)
+        records = get_all_dns_history(query, type: "a")
+        records.map do |record|
+          (record["values"] || []).map { |value| value["ip"] }
+        end.flatten.compact.uniq
+      end
+
+      #
+      # IP search
+      #
+      # @param [String] query
+      #
+      # @return [Array<Mihari::Artifact>]
+      #
+      def ip_search(query)
+        records = search_by_ip(query)
+        records.filter_map do |record|
+          data = record["hostname"]
+          Artifact.new(data: data, metadata: record)
+        end
+      end
+
+      #
+      # Mail search
+      #
+      # @param [String] query
+      #
+      # @return [Array<String>]
+      #
+      def mail_search(query)
+        records = search_by_mail(query)
+        records.filter_map do |record|
+          data = record["hostname"]
+          Artifact.new(data: data, metadata: record)
+        end
+      end
+
       #
       # @param [String] mail
       #

data/lib/mihari/clients/shodan.rb

@@ -3,6 +3,8 @@
 module Mihari
   module Clients
     class Shodan < Base
+      PAGE_SIZE = 100
+
       # @return [String]
       attr_reader :api_key
 
@@ -10,11 +12,12 @@ module Mihari
       # @param [String] base_url
       # @param [String, nil] api_key
       # @param [Hash] headers
+      # @param [Integer, nil] interval
       #
-      def initialize(base_url = "https://api.shodan.io", api_key:, headers: {})
+      def initialize(base_url = "https://api.shodan.io", api_key:, headers: {}, interval: nil)
         raise(ArgumentError, "'api_key' argument is required") unless api_key
 
-        super(base_url, headers: headers)
+        super(base_url, headers: headers, interval: interval)
 
         @api_key = api_key
       end
@@ -34,7 +37,32 @@ module Mihari
           key: api_key
         }
         res = get("/shodan/host/search", params: params)
-        Structs::Shodan::Result.from_dynamic! JSON.parse(res.body.to_s)
+        Structs::Shodan::Response.from_dynamic! JSON.parse(res.body.to_s)
+      end
+
+      #
+      # @param [String] query
+      # @param [Boolean] minify
+      # @param [Integer] pagination_limit
+      #
+      # @return [Enumerable<Structs::Shodan::Response>]
+      #
+      def search_with_pagination(query, minify: true, pagination_limit: Mihari.config.pagination_limit)
+        Enumerator.new do |y|
+          (1..pagination_limit).each do |page|
+            res = search(query, page: page, minify: minify)
+
+            y.yield res
+
+            break if res.total <= page * PAGE_SIZE
+
+            sleep_interval
+          rescue JSON::ParserError
+            # ignore JSON::ParserError
+            # ref. https://github.com/ninoseki/mihari/issues/197
+            next
+          end
+        end
       end
     end
   end

data/lib/mihari/clients/urlscan.rb

@@ -7,26 +7,52 @@ module Mihari
       # @param [String] base_url
       # @param [String, nil] api_key
       # @param [Hash] headers
+      # @param [Interval, nil] interval
       #
-      def initialize(base_url = "https://urlscan.io", api_key:, headers: {})
+      def initialize(base_url = "https://urlscan.io", api_key:, headers: {}, interval: nil)
         raise(ArgumentError, "'api_key' argument is required") if api_key.nil?
 
         headers["api-key"] = api_key
 
-        super(base_url, headers: headers)
+        super(base_url, headers: headers, interval: interval)
       end
 
       #
       # @param [String] q
-      # @param [Integer] size
+      # @param [Integer, nil] size
       # @param [String, nil] search_after
       #
-      # @return [Hash]
+      # @return [Structs::Urlscan::Response]
       #
-      def search(q, size: 100, search_after: nil)
+      def search(q, size: nil, search_after: nil)
         params = { q: q, size: size, search_after: search_after }.compact
         res = get("/api/v1/search/", params: params)
-        JSON.parse res.body.to_s
+        Structs::Urlscan::Response.from_dynamic! JSON.parse(res.body.to_s)
+      end
+
+      #
+      # @param [String] q
+      # @param [Integer, nil] size
+      # @param [Integer] pagination_limit
+      #
+      # @return [Enumerable<Structs::Urlscan::Response>]
+      #
+      def search_with_pagination(q, size: nil, pagination_limit: Mihari.config.pagination_limit)
+        search_after = nil
+
+        Enumerator.new do |y|
+          pagination_limit.times do
+            res = search(q, size: size, search_after: search_after)
+
+            y.yield res
+
+            break unless res.has_more
+
+            search_after = res.results.last.sort.join(",")
+
+            sleep_interval
+          end
+        end
       end
     end
   end

data/lib/mihari/clients/virustotal.rb

@@ -7,13 +7,14 @@ module Mihari
       # @param [String] base_url
       # @param [String, nil] api_key
       # @param [Hash] headers
+      # @param [Integer, nil] interval
       #
-      def initialize(base_url = "https://www.virustotal.com", api_key:, headers: {})
+      def initialize(base_url = "https://www.virustotal.com", api_key:, headers: {}, interval: nil)
         raise(ArgumentError, "'api_key' argument is required") if api_key.nil?
 
         headers["x-apikey"] = api_key
 
-        super(base_url, headers: headers)
+        super(base_url, headers: headers, interval: interval)
       end
 
       #
@@ -38,11 +39,35 @@ module Mihari
       # @param [String] query
       # @param [String, nil] cursor
       #
-      # @return [Hash]
+      # @return [Structs::VirusTotalIntelligence::Response]
       #
       def intel_search(query, cursor: nil)
         params = { query: query, cursor: cursor }.compact
-        _get("/api/v3/intelligence/search", params: params)
+        res = _get("/api/v3/intelligence/search", params: params)
+        Structs::VirusTotalIntelligence::Response.from_dynamic! res
+      end
+
+      #
+      # @param [String] query
+      # @param [Integer] pagination_limit
+      #
+      # @return [Enumerable<Structs::VirusTotalIntelligence::Response>]
+      #
+      def intel_search_with_pagination(query, pagination_limit: Mihari.config.pagination_limit)
+        cursor = nil
+
+        Enumerator.new do |y|
+          pagination_limit.times do
+            res = intel_search(query, cursor: cursor)
+
+            y.yield res
+
+            cursor = res.meta.cursor
+            break if cursor.nil?
+
+            sleep_interval
+          end
+        end
       end
 
       private

data/lib/mihari/clients/zoomeye.rb

@@ -3,18 +3,21 @@
 module Mihari
   module Clients
     class ZoomEye < Base
+      PAGE_SIZE = 10
+
       attr_reader :api_key
 
       #
       # @param [String] base_url
       # @param [String, nil] api_key
       # @param [Hash] headers
+      # @param [Integer, nil] interval
       #
-      def initialize(base_url = "https://api.zoomeye.org", api_key:, headers: {})
+      def initialize(base_url = "https://api.zoomeye.org", api_key:, headers: {}, interval: nil)
         raise(ArgumentError, "'api_key' argument is required") unless api_key
 
         headers["api-key"] = api_key
-        super(base_url, headers: headers)
+        super(base_url, headers: headers, interval: interval)
       end
 
       #
@@ -36,6 +39,30 @@ module Mihari
         _get("/host/search", params: params)
       end
 
+      #
+      # @param [String] query
+      # @param [String, nil] facets
+      # @param [Integer] pagination_limit
+      #
+      # @return [Enumerable<Hash>]
+      #
+      def host_search_with_pagination(query, facets: nil, pagination_limit: Mihari.config.pagination_limit)
+        Enumerator.new do |y|
+          (1..pagination_limit).each do |page|
+            res = host_search(query, facets: facets, page: page)
+
+            break if res.nil?
+
+            y.yield res
+
+            total = res["total"].to_i
+            break if total <= page * PAGE_SIZE
+
+            sleep_interval
+          end
+        end
+      end
+
       #
       # Search the Web technologies
       #
@@ -55,6 +82,30 @@ module Mihari
         _get("/web/search", params: params)
       end
 
+      #
+      # @param [String] query
+      # @param [String, nil] facets
+      # @param [Integer] pagination_limit
+      #
+      # @return [Enumerable<Hash>]
+      #
+      def web_search_with_pagination(query, facets: nil, pagination_limit: Mihari.config.pagination_limit)
+        Enumerator.new do |y|
+          (1..pagination_limit).each do |page|
+            res = web_search(query, facets: facets, page: page)
+
+            break if res.nil?
+
+            y.yield res
+
+            total = res["total"].to_i
+            break if total <= page * PAGE_SIZE
+
+            sleep_interval
+          end
+        end
+      end
+
       private
 
       #