mihari 5.1.1 → 5.1.2

data/lib/mihari/analyzers/shodan.rb

@@ -10,6 +10,12 @@ module Mihari
       # @return [String, nil]
       attr_reader :api_key
 
+      # @return [Integer]
+      attr_reader :interval
+
+      # @return [String]
+      attr_reader :query
+
       def initialize(*args, **kwargs)
         super(*args, **kwargs)
 
@@ -18,13 +24,9 @@ module Mihari
 
       def artifacts
         results = search
-        return [] unless results || results.empty?
-
-        results = results.map { |result| Structs::Shodan::Result.from_dynamic!(result) }
-        matches = results.map { |result| result.matches || [] }.flatten
+        return [] if results.empty?
 
-        uniq_matches = matches.uniq(&:ip_str)
-        uniq_matches.map { |match| build_artifact(match, matches) }
+        results.map { |result| result.to_artifacts(source) }.flatten.uniq(&:data)
       end
 
       private
@@ -42,29 +44,25 @@ module Mihari
       #
       # Search with pagination
       #
-      # @param [String] query
       # @param [Integer] page
       #
-      # @return [Hash]
+      # @return [Structs::Shodan::Result]
       #
-      def search_with_page(query, page: 1)
+      def search_with_page(page: 1)
         client.search(query, page: page)
       end
 
       #
       # Search
       #
-      # @return [Array<Hash>]
+      # @return [Array<Structs::Shodan::Result>]
       #
       def search
         responses = []
         (1..Float::INFINITY).each do |page|
-          res = search_with_page(query, page: page)
-
-          break unless res
-
+          res = search_with_page(page: page)
           responses << res
-          break if res["total"].to_i <= page * PAGE_SIZE
+          break if res.total <= page * PAGE_SIZE
 
           # sleep #{interval} seconds to avoid the rate limitation (if it is set)
           sleep interval
@@ -76,42 +74,6 @@ module Mihari
         responses
       end
 
-      #
-      # Collect metadata from matches
-      #
-      # @param [Array<Structs::Shodan::Match>] matches
-      # @param [String] ip
-      #
-      # @return [Array<Hash>]
-      #
-      def collect_metadata_by_ip(matches, ip)
-        matches.select { |match| match.ip_str == ip }.map(&:metadata)
-      end
-
-      #
-      # Collect ports from matches
-      #
-      # @param [Array<Structs::Shodan::Match>] matches
-      # @param [String] ip
-      #
-      # @return [Array<String>]
-      #
-      def collect_ports_by_ip(matches, ip)
-        matches.select { |match| match.ip_str == ip }.map(&:port)
-      end
-
-      #
-      # Collect hostnames from matches
-      #
-      # @param [Array<Structs::Shodan::Match>] matches
-      # @param [String] ip
-      #
-      # @return [Array<String>]
-      #
-      def collect_hostnames_by_ip(matches, ip)
-        matches.select { |match| match.ip_str == ip }.map(&:hostnames).flatten.uniq
-      end
-
       #
       # Build an artifact from a Shodan search API response
       #
@@ -121,36 +83,6 @@ module Mihari
       # @return [Artifact]
       #
       def build_artifact(match, matches)
-        as = nil
-        as = AutonomousSystem.new(asn: normalize_asn(match.asn)) unless match.asn.nil?
-
-        geolocation = nil
-        if !match.location.country_name.nil? && !match.location.country_code.nil?
-          geolocation = Geolocation.new(
-            country: match.location.country_name,
-            country_code: match.location.country_code
-          )
-        end
-
-        metadata = collect_metadata_by_ip(matches, match.ip_str)
-
-        ports = collect_ports_by_ip(matches, match.ip_str).map do |port|
-          Port.new(port: port)
-        end
-
-        reverse_dns_names = collect_hostnames_by_ip(matches, match.ip_str).map do |name|
-          ReverseDnsName.new(name: name)
-        end
-
-        Artifact.new(
-          data: match.ip_str,
-          source: source,
-          metadata: metadata,
-          autonomous_system: as,
-          geolocation: geolocation,
-          ports: ports,
-          reverse_dns_names: reverse_dns_names
-        )
       end
     end
   end

data/lib/mihari/analyzers/urlscan.rb

@@ -15,6 +15,15 @@ module Mihari
       # @return [String, nil]
       attr_reader :api_key
 
+      # @return [String]
+      attr_reader :query
+
+      # @return [Integer]
+      attr_reader :interval
+
+      # @return [String]
+      attr_reader :allowed_data_types
+
       def initialize(*args, **kwargs)
         super
 

data/lib/mihari/analyzers/virustotal.rb

@@ -7,11 +7,15 @@ module Mihari
 
       param :query
 
+      # @return [String]
       attr_reader :type
 
       # @return [String, nil]
       attr_reader :api_key
 
+      # @return [String]
+      attr_reader :query
+
       def initialize(*args, **kwargs)
         super(*args, **kwargs)
 

data/lib/mihari/analyzers/virustotal_intelligence.rb

@@ -10,6 +10,12 @@ module Mihari
       # @return [String, nil]
       attr_reader :api_key
 
+      # @return [String]
+      attr_reader :query
+
+      # @return [Integer]
+      attr_reader :interval
+
       def initialize(*args, **kwargs)
         super
 
@@ -19,7 +25,7 @@ module Mihari
       end
 
       def artifacts
-        responses = search_witgh_cursor
+        responses = search_with_cursor
         responses.map do |response|
           response.data.map do |datum|
             Artifact.new(data: datum.value, source: source, metadata: datum.metadata)
@@ -47,7 +53,7 @@ module Mihari
       #
       # @return [Array<Structs::VirusTotalIntelligence::Response>]
       #
-      def search_witgh_cursor
+      def search_with_cursor
         cursor = nil
         responses = []
 

data/lib/mihari/analyzers/zoomeye.rb

@@ -12,6 +12,15 @@ module Mihari
       # @return [String, nil]
       attr_reader :api_key
 
+      # @return [String]
+      attr_reader :query
+
+      # @return [String]
+      attr_reader :type
+
+      # @return [Integer]
+      attr_reader :interval
+
       def initialize(*args, **kwargs)
         super(*args, **kwargs)
 

data/lib/mihari/clients/binaryedge.rb

@@ -3,6 +3,11 @@
 module Mihari
   module Clients
     class BinaryEdge < Base
+      #
+      # @param [String] base_url
+      # @param [String, nil] api_key
+      # @param [Hash] headers
+      #
       def initialize(base_url = "https://api.binaryedge.io/v2", api_key:, headers: {})
         raise(ArgumentError, "'api_key' argument is required") unless api_key
 

data/lib/mihari/clients/censys.rb

@@ -7,8 +7,8 @@ module Mihari
     class Censys < Base
       #
       # @param [String] base_url
-      # @param [String] id
-      # @param [String] secret
+      # @param [String, nil] id
+      # @param [String, nil] secret
       # @param [Hash] headers
       #
       def initialize(base_url = "https://search.censys.io", id:, secret:, headers: {})
@@ -30,12 +30,12 @@ module Mihari
       # @params [Integer, nil] per_page the number of results to be returned for each page.
       # @params [Integer, nil] cursor the cursor of the desired result set.
       #
-      # @return [Hash]
+      # @return [Structs::Censys::Response]
       #
       def search(query, per_page: nil, cursor: nil)
         params = { q: query, per_page: per_page, cursor: cursor }.compact
         res = get("/api/v2/hosts/search", params: params)
-        JSON.parse(res.body.to_s)
+        Structs::Censys::Response.from_dynamic! JSON.parse(res.body.to_s)
       end
     end
   end

data/lib/mihari/clients/circl.rb

@@ -7,8 +7,8 @@ module Mihari
     class CIRCL < Base
       #
       # @param [String] base_url
-      # @param [String] username
-      # @param [String] password
+      # @param [String, nil] username
+      # @param [String, nil] password
       # @param [Hash] headers
       #
       def initialize(base_url = "https://www.circl.lu", username:, password:, headers: {})
@@ -43,7 +43,7 @@ module Mihari
       #
       #
       # @param [String] path
-      # @param [Array<Hash>] params
+      # @param [Hash] params
       #
       def _get(path, params: {})
         res = get(path, params: params)

data/lib/mihari/clients/greynoise.rb

@@ -3,6 +3,11 @@
 module Mihari
   module Clients
     class GreyNoise < Base
+      #
+      # @param [String] base_url
+      # @param [String, nil] api_key
+      # @param [Hash] headers
+      #
       def initialize(base_url = "https://api.greynoise.io", api_key:, headers: {})
         raise(ArgumentError, "'api_key' argument is required") unless api_key
 
@@ -22,7 +27,7 @@ module Mihari
       def gnql_search(query, size: nil, scroll: nil)
         params = { query: query, size: size, scroll: scroll }.compact
         res = get("/v2/experimental/gnql", params: params)
-        JSON.parse res.body.to_s
+        Structs::GreyNoise::Response.from_dynamic! JSON.parse(res.body.to_s)
       end
     end
   end

data/lib/mihari/clients/misp.rb

@@ -5,7 +5,7 @@ module Mihari
     class MISP < Base
       #
       # @param [String] base_url
-      # @param [String] api_key
+      # @param [String, nil] api_key
       # @param [Hash] headers
       #
       def initialize(base_url, api_key:, headers: {})
@@ -15,6 +15,11 @@ module Mihari
         super(base_url, headers: headers)
       end
 
+      #
+      # @param [Hash] payload
+      #
+      # @return [Hash]
+      #
       def create_event(payload)
         res = post("/events/add", json: payload)
         JSON.parse(res.body.to_s)

data/lib/mihari/clients/onyphe.rb

@@ -3,8 +3,14 @@
 module Mihari
   module Clients
     class Onyphe < Base
+      # @return [String]
       attr_reader :api_key
 
+      #
+      # @param [String] base_url
+      # @param [String, nil] api_key
+      # @param [Hash] headers
+      #
       def initialize(base_url = "https://www.onyphe.io", api_key:, headers: {})
         raise(ArgumentError, "'api_key' argument is required") if api_key.nil?
 
@@ -13,10 +19,16 @@ module Mihari
         @api_key = api_key
       end
 
+      #
+      # @param [String] query
+      # @param [Integer] page
+      #
+      # @return [Hash]
+      #
       def datascan(query, page: 1)
         params = { page: page, apikey: api_key }
         res = get("/api/v2/simple/datascan/#{query}", params: params)
-        JSON.parse(res.body.to_s)
+        Structs::Onyphe::Response.from_dynamic! JSON.parse(res.body.to_s)
       end
     end
   end

data/lib/mihari/clients/otx.rb

@@ -3,6 +3,11 @@
 module Mihari
   module Clients
     class OTX < Base
+      #
+      # @param [String] base_url
+      # @param [String, nil] api_key
+      # @param [Hash] headers
+      #
       def initialize(base_url = "https://otx.alienvault.com", api_key:, headers: {})
         raise(ArgumentError, "'api_key' argument is required") unless api_key
 
@@ -10,16 +15,31 @@ module Mihari
         super(base_url, headers: headers)
       end
 
+      #
+      # @param [String] ip
+      #
+      # @return [Hash]
+      #
       def query_by_ip(ip)
         _get "/api/v1/indicators/IPv4/#{ip}/passive_dns"
       end
 
+      #
+      # @param [String] domain
+      #
+      # @return [Hash]
+      #
       def query_by_domain(domain)
         _get "/api/v1/indicators/domain/#{domain}/passive_dns"
       end
 
       private
 
+      #
+      # @param [String] path
+      #
+      # @return [Hash]
+      #
       def _get(path)
         res = get(path)
         JSON.parse(res.body.to_s)

data/lib/mihari/clients/passivetotal.rb

@@ -7,8 +7,8 @@ module Mihari
     class PassiveTotal < Base
       #
       # @param [String] base_url
-      # @param [String] username
-      # @param [String] api_key
+      # @param [String, nil] username
+      # @param [String, nil] api_key
       # @param [Hash] headers
       #
       def initialize(base_url = "https://api.passivetotal.org", username:, api_key:, headers: {})
@@ -31,6 +31,8 @@ module Mihari
       #
       # @param [String] query
       #
+      # @return [Hash]
+      #
       def passive_dns_search(query)
         params = { query: query }
         _get("/v2/dns/passive/unique", params: params)
@@ -56,6 +58,8 @@ module Mihari
       # @param [String] path
       # @param [Hash] params
       #
+      # @return [Hash]
+      #
       def _get(path, params: {})
         res = get(path, params: params)
         JSON.parse(res.body.to_s)

data/lib/mihari/clients/publsedive.rb

@@ -3,8 +3,14 @@
 module Mihari
   module Clients
     class PulseDive < Base
+      # @return [String]
       attr_reader :api_key
 
+      #
+      # @param [String] base_url
+      # @param [String, nil] api_key
+      # @param [Hash] headers
+      #
       def initialize(base_url = "https://pulsedive.com", api_key:, headers: {})
         super(base_url, headers: headers)
 
@@ -13,21 +19,32 @@ module Mihari
         raise(ArgumentError, "'api_key' argument is required") unless api_key
       end
 
+      #
+      # @param [String] indicator_id
+      #
+      # @return [Hash]
+      #
       def get_indicator(ip_or_domain)
         _get "/api/info.php", params: { indicator: ip_or_domain }
       end
 
+      #
+      # @param [String] indicator_id
+      #
+      # @return [Hash]
+      #
       def get_properties(indicator_id)
         _get "/api/info.php", params: { iid: indicator_id, get: "properties" }
       end
 
       private
 
-      #
       #
       # @param [String] path
       # @param [Hash] params
       #
+      # @return [Hash]
+      #
       def _get(path, params: {})
         params["key"] = api_key
 

data/lib/mihari/clients/securitytrails.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Clients
+    class SecurityTrails < Base
+      #
+      # @param [String] base_url
+      # @param [String, nil] api_key
+      # @param [Hash] headers
+      #
+      def initialize(base_url = "https://api.securitytrails.com", api_key:, headers: {})
+        raise(ArgumentError, "'api_key' argument is required") unless api_key
+
+        headers["apikey"] = api_key
+        super(base_url, headers: headers)
+      end
+
+      #
+      # @param [String] mail
+      #
+      # @return [Array<Hash>]
+      #
+      def search_by_mail(mail)
+        res = _post "/v1/domains/list", json: { filter: { whois_email: mail } }
+        res["records"] || []
+      end
+
+      #
+      # @param [String] ip
+      #
+      # @return [Array<Hash>]
+      #
+      def search_by_ip(ip)
+        res = _post "/v1/domains/list", json: { filter: { ipv4: ip } }
+        res["records"] || []
+      end
+
+      #
+      # @param [String] domain
+      # @param [String] type
+      #
+      # @return [Array<Hash>]
+      #
+      def get_all_dns_history(domain, type:)
+        first_page = get_dns_history(domain, type: type, page: 1)
+
+        pages = first_page["pages"].to_i
+        records = first_page["records"] || []
+
+        (2..pages).each do |page_idx|
+          next_page = get_dns_history(domain, type: type, page: page_idx)
+          records << next_page["records"]
+        end
+
+        records.flatten
+      end
+
+      private
+
+      #
+      # @param [String] domain
+      # @param [String] type
+      # @param [Integer] page
+      #
+      # @return [Array<Hash>]
+      #
+      def get_dns_history(domain, type:, page:)
+        _get "/v1/history/#{domain}/dns/#{type}", params: { page: page }
+      end
+
+      #
+      # @param [String] path
+      # @param [Hash, nil] params
+      #
+      # @return [Hash]
+      #
+      def _get(path, params:)
+        res = get(path, params: params)
+        JSON.parse(res.body.to_s)
+      end
+
+      #
+      # @param [String] path
+      # @param [Hash, nil] json
+      #
+      # @return [Hash]
+      #
+      def _post(path, json:)
+        res = post(path, json: json)
+        JSON.parse(res.body.to_s)
+      end
+    end
+  end
+end

data/lib/mihari/clients/shodan.rb

@@ -3,8 +3,14 @@
 module Mihari
   module Clients
     class Shodan < Base
+      # @return [String]
       attr_reader :api_key
 
+      #
+      # @param [String] base_url
+      # @param [String, nil] api_key
+      # @param [Hash] headers
+      #
       def initialize(base_url = "https://api.shodan.io", api_key:, headers: {})
         raise(ArgumentError, "'api_key' argument is required") unless api_key
 
@@ -13,8 +19,13 @@ module Mihari
         @api_key = api_key
       end
 
-      # Search Shodan using the same query syntax as the website and use facets
-      # to get summary information for different properties.
+      #
+      # @param [String] query
+      # @param [Integer] page
+      # @param [Boolean] minify
+      #
+      # @return [Structs::Shodan::Result]
+      #
       def search(query, page: 1, minify: true)
         params = {
           query: query,
@@ -23,7 +34,7 @@ module Mihari
           key: api_key
         }
         res = get("/shodan/host/search", params: params)
-        JSON.parse(res.body.to_s)
+        Structs::Shodan::Result.from_dynamic! JSON.parse(res.body.to_s)
       end
     end
   end

data/lib/mihari/clients/the_hive.rb

@@ -5,7 +5,7 @@ module Mihari
     class TheHive < Base
       #
       # @param [String] base_url
-      # @param [String] api_key
+      # @param [String, nil] api_key
       # @param [String, nil] api_version
       # @param [Hash] headers
       #
@@ -18,6 +18,11 @@ module Mihari
         super(base_url, headers: headers)
       end
 
+      #
+      # @param [Hash] json
+      #
+      # @return [Hash]
+      #
       def alert(json)
         json = json.to_camelback_keys.compact
         res = post("/alert", json: json)

data/lib/mihari/clients/urlscan.rb

@@ -5,7 +5,7 @@ module Mihari
     class UrlScan < Base
       #
       # @param [String] base_url
-      # @param [String] api_key
+      # @param [String, nil] api_key
       # @param [Hash] headers
       #
       def initialize(base_url = "https://urlscan.io", api_key:, headers: {})
@@ -21,6 +21,8 @@ module Mihari
       # @param [Integer] size
       # @param [String, nil] search_after
       #
+      # @return [Hash]
+      #
       def search(q, size: 100, search_after: nil)
         params = { q: q, size: size, search_after: search_after }.compact
         res = get("/api/v1/search/", params: params)