mihari 5.1.0 → 5.1.2

data/lib/mihari/analyzers/passivetotal.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "passivetotal"
-
 module Mihari
   module Analyzers
     class PassiveTotal < Base
@@ -18,6 +16,9 @@ module Mihari
       # @return [String, nil]
       attr_reader :api_key
 
+      # @return [String]
+      attr_reader :query
+
       def initialize(*args, **kwargs)
         super(*args, **kwargs)
 
@@ -42,8 +43,8 @@ module Mihari
         %w[passivetotal_username passivetotal_api_key]
       end
 
-      def api
-        @api ||= ::PassiveTotal::API.new(username: username, api_key: api_key)
+      def client
+        @client ||= Clients::PassiveTotal.new(username: username, api_key: api_key)
       end
 
       #
@@ -79,7 +80,7 @@ module Mihari
       # @return [Array<String>]
       #
       def passive_dns_search
-        res = api.dns.passive_unique(query)
+        res = client.passive_dns_search(query)
         res["results"] || []
       end
 
@@ -89,7 +90,7 @@ module Mihari
       # @return [Array<Mihari::Artifact>]
       #
       def reverse_whois_search
-        res = api.whois.search(query: query, field: "email")
+        res = client.reverse_whois_search(query: query, field: "email")
         results = res["results"] || []
         results.map do |result|
           data = result["domain"]
@@ -103,7 +104,7 @@ module Mihari
       # @return [Array<Mihari::Artifact>]
       #
       def ssl_search
-        res = api.ssl.history(query)
+        res = client.ssl_search(query)
         results = res["results"] || []
         results.map do |result|
           data = result["ipAddresses"]

data/lib/mihari/analyzers/pulsedive.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "pulsedive"
-
 module Mihari
   module Analyzers
     class Pulsedive < Base
@@ -15,6 +13,9 @@ module Mihari
       # @return [String, nil]
       attr_reader :api_key
 
+      # @return [Integer]
+      attr_reader :query
+
       def initialize(*args, **kwargs)
         super
 
@@ -34,8 +35,8 @@ module Mihari
         %w[pulsedive_api_key]
       end
 
-      def api
-        @api ||= ::Pulsedive::API.new(api_key)
+      def client
+        @client ||= Clients::PulseDive.new(api_key: api_key)
       end
 
       #
@@ -55,12 +56,12 @@ module Mihari
       def search
         raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
 
-        indicator = api.indicator.get_by_value(query)
+        indicator = client.get_indicator(query)
         iid = indicator["iid"]
 
-        properties = api.indicator.get_properties_by_id(iid)
+        properties = client.get_properties(iid)
         (properties["dns"] || []).filter_map do |property|
-          if ["A", "PTR"].include?(property["name"])
+          if %w[A PTR].include?(property["name"])
             nil
           else
             data = property["value"]

data/lib/mihari/analyzers/rule.rb

@@ -7,7 +7,6 @@ module Mihari
       "censys" => Censys,
       "circl" => CIRCL,
       "crtsh" => Crtsh,
-      "dnpedia" => DNPedia,
       "dnstwister" => DNSTwister,
       "feed" => Feed,
       "greynoise" => GreyNoise,

data/lib/mihari/analyzers/securitytrails.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "securitytrails"
-
 module Mihari
   module Analyzers
     class SecurityTrails < Base
@@ -15,6 +13,9 @@ module Mihari
       # @return [String, nil]
       attr_reader :api_key
 
+      # @return [String]
+      attr_reader :query
+
       def initialize(*args, **kwargs)
         super
 
@@ -34,8 +35,8 @@ module Mihari
         %w[securitytrails_api_key]
       end
 
-      def api
-        @api ||= ::SecurityTrails::API.new(api_key)
+      def client
+        @client ||= Clients::SecurityTrails.new(api_key: api_key)
       end
 
       #
@@ -71,8 +72,7 @@ module Mihari
       # @return [Array<String>]
       #
       def domain_search
-        result = api.history.get_all_dns_history(query, type: "a")
-        records = result["records"] || []
+        records = client.get_all_dns_history(query, type: "a")
         records.map do |record|
           (record["values"] || []).map { |value| value["ip"] }
         end.flatten.compact.uniq
@@ -84,8 +84,7 @@ module Mihari
       # @return [Array<Mihari::Artifact>]
       #
       def ip_search
-        result = api.domains.search(filter: { ipv4: query })
-        records = result["records"] || []
+        records = client.search_by_ip(query)
         records.filter_map do |record|
           data = record["hostname"]
           Artifact.new(data: data, source: source, metadata: record)
@@ -98,8 +97,7 @@ module Mihari
       # @return [Array<String>]
       #
       def mail_search
-        result = api.domains.search(filter: { whois_email: query })
-        records = result["records"] || []
+        records = client.search_by_mail(query)
         records.filter_map do |record|
           data = record["hostname"]
           Artifact.new(data: data, source: source, metadata: record)

data/lib/mihari/analyzers/shodan.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "shodan"
-
 module Mihari
   module Analyzers
     class Shodan < Base
@@ -12,6 +10,12 @@ module Mihari
       # @return [String, nil]
       attr_reader :api_key
 
+      # @return [Integer]
+      attr_reader :interval
+
+      # @return [String]
+      attr_reader :query
+
       def initialize(*args, **kwargs)
         super(*args, **kwargs)
 
@@ -20,13 +24,9 @@ module Mihari
 
       def artifacts
         results = search
-        return [] unless results || results.empty?
+        return [] if results.empty?
 
-        results = results.map { |result| Structs::Shodan::Result.from_dynamic!(result) }
-        matches = results.map { |result| result.matches || [] }.flatten
-
-        uniq_matches = matches.uniq(&:ip_str)
-        uniq_matches.map { |match| build_artifact(match, matches) }
+        results.map { |result| result.to_artifacts(source) }.flatten.uniq(&:data)
       end
 
       private
@@ -37,40 +37,32 @@ module Mihari
         %w[shodan_api_key]
       end
 
-      def api
-        @api ||= ::Shodan::API.new(key: api_key)
+      def client
+        @client ||= Clients::Shodan.new(api_key: api_key)
       end
 
       #
       # Search with pagination
       #
-      # @param [String] query
       # @param [Integer] page
       #
-      # @return [Hash]
+      # @return [Structs::Shodan::Result]
       #
-      def search_with_page(query, page: 1)
-        api.host.search(query, page: page)
-      rescue ::Shodan::Error => e
-        raise RetryableError, e if e.message.include?("request timed out")
-
-        raise e
+      def search_with_page(page: 1)
+        client.search(query, page: page)
       end
 
       #
       # Search
       #
-      # @return [Array<Hash>]
+      # @return [Array<Structs::Shodan::Result>]
       #
       def search
         responses = []
         (1..Float::INFINITY).each do |page|
-          res = search_with_page(query, page: page)
-
-          break unless res
-
+          res = search_with_page(page: page)
           responses << res
-          break if res["total"].to_i <= page * PAGE_SIZE
+          break if res.total <= page * PAGE_SIZE
 
           # sleep #{interval} seconds to avoid the rate limitation (if it is set)
           sleep interval
@@ -82,42 +74,6 @@ module Mihari
         responses
       end
 
-      #
-      # Collect metadata from matches
-      #
-      # @param [Array<Structs::Shodan::Match>] matches
-      # @param [String] ip
-      #
-      # @return [Array<Hash>]
-      #
-      def collect_metadata_by_ip(matches, ip)
-        matches.select { |match| match.ip_str == ip }.map(&:metadata)
-      end
-
-      #
-      # Collect ports from matches
-      #
-      # @param [Array<Structs::Shodan::Match>] matches
-      # @param [String] ip
-      #
-      # @return [Array<String>]
-      #
-      def collect_ports_by_ip(matches, ip)
-        matches.select { |match| match.ip_str == ip }.map(&:port)
-      end
-
-      #
-      # Collect hostnames from matches
-      #
-      # @param [Array<Structs::Shodan::Match>] matches
-      # @param [String] ip
-      #
-      # @return [Array<String>]
-      #
-      def collect_hostnames_by_ip(matches, ip)
-        matches.select { |match| match.ip_str == ip }.map(&:hostnames).flatten.uniq
-      end
-
       #
       # Build an artifact from a Shodan search API response
       #
@@ -127,36 +83,6 @@ module Mihari
       # @return [Artifact]
       #
       def build_artifact(match, matches)
-        as = nil
-        as = AutonomousSystem.new(asn: normalize_asn(match.asn)) unless match.asn.nil?
-
-        geolocation = nil
-        if !match.location.country_name.nil? && !match.location.country_code.nil?
-          geolocation = Geolocation.new(
-            country: match.location.country_name,
-            country_code: match.location.country_code
-          )
-        end
-
-        metadata = collect_metadata_by_ip(matches, match.ip_str)
-
-        ports = collect_ports_by_ip(matches, match.ip_str).map do |port|
-          Port.new(port: port)
-        end
-
-        reverse_dns_names = collect_hostnames_by_ip(matches, match.ip_str).map do |name|
-          ReverseDnsName.new(name: name)
-        end
-
-        Artifact.new(
-          data: match.ip_str,
-          source: source,
-          metadata: metadata,
-          autonomous_system: as,
-          geolocation: geolocation,
-          ports: ports,
-          reverse_dns_names: reverse_dns_names
-        )
       end
     end
   end

data/lib/mihari/analyzers/urlscan.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "urlscan"
-
 module Mihari
   module Analyzers
     class Urlscan < Base
@@ -17,10 +15,22 @@ module Mihari
       # @return [String, nil]
       attr_reader :api_key
 
+      # @return [String]
+      attr_reader :query
+
+      # @return [Integer]
+      attr_reader :interval
+
+      # @return [String]
+      attr_reader :allowed_data_types
+
       def initialize(*args, **kwargs)
         super
 
-        raise InvalidInputError, "allowed_data_types should be any of url, domain and ip." unless valid_alllowed_data_types?
+        unless valid_alllowed_data_types?
+          raise InvalidInputError,
+            "allowed_data_types should be any of url, domain and ip."
+        end
 
         @api_key = kwargs[:api_key] || Mihari.config.urlscan_api_key
       end
@@ -44,8 +54,8 @@ module Mihari
         %w[urlscan_api_key]
       end
 
-      def api
-        @api ||= ::UrlScan::API.new(api_key)
+      def client
+        @client ||= Clients::UrlScan.new(api_key: api_key)
       end
 
       #
@@ -54,7 +64,7 @@ module Mihari
       # @return [Structs::Urlscan::Response]
       #
       def search_with_search_after(search_after: nil)
-        res = api.search(query, size: SIZE, search_after: search_after)
+        res = client.search(query, size: SIZE, search_after: search_after)
         Structs::Urlscan::Response.from_dynamic! res
       end
 

data/lib/mihari/analyzers/virustotal.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "virustotal"
-
 module Mihari
   module Analyzers
     class VirusTotal < Base
@@ -9,11 +7,15 @@ module Mihari
 
       param :query
 
+      # @return [String]
       attr_reader :type
 
       # @return [String, nil]
       attr_reader :api_key
 
+      # @return [String]
+      attr_reader :query
+
       def initialize(*args, **kwargs)
         super(*args, **kwargs)
 
@@ -33,8 +35,8 @@ module Mihari
         %w[virustotal_api_key]
       end
 
-      def api
-        @api = ::VirusTotal::API.new(key: api_key)
+      def client
+        @client = Clients::VirusTotal.new(api_key: api_key)
       end
 
       #
@@ -68,7 +70,7 @@ module Mihari
       # @return [Array<Mihari::Artifact>]
       #
       def domain_search
-        res = api.domain.resolutions(query)
+        res = client.domain_search(query)
 
         data = res["data"] || []
         data.filter_map do |item|
@@ -83,7 +85,7 @@ module Mihari
       # @return [Array<Mihari::Artifact>]
       #
       def ip_search
-        res = api.ip_address.resolutions(query)
+        res = client.ip_search(query)
 
         data = res["data"] || []
         data.filter_map do |item|

data/lib/mihari/analyzers/virustotal_intelligence.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "virustotal"
-
 module Mihari
   module Analyzers
     class VirusTotalIntelligence < Base
@@ -12,6 +10,12 @@ module Mihari
       # @return [String, nil]
       attr_reader :api_key
 
+      # @return [String]
+      attr_reader :query
+
+      # @return [Integer]
+      attr_reader :interval
+
       def initialize(*args, **kwargs)
         super
 
@@ -21,7 +25,7 @@ module Mihari
       end
 
       def artifacts
-        responses = search_witgh_cursor
+        responses = search_with_cursor
         responses.map do |response|
           response.data.map do |datum|
             Artifact.new(data: datum.value, source: source, metadata: datum.metadata)
@@ -40,8 +44,8 @@ module Mihari
       #
       # @return [::VirusTotal::API]
       #
-      def api
-        @api = ::VirusTotal::API.new(key: api_key)
+      def client
+        @client = Clients::VirusTotal.new(api_key: api_key)
       end
 
       #
@@ -49,12 +53,13 @@ module Mihari
       #
       # @return [Array<Structs::VirusTotalIntelligence::Response>]
       #
-      def search_witgh_cursor
+      def search_with_cursor
         cursor = nil
         responses = []
 
         loop do
-          response = Structs::VirusTotalIntelligence::Response.from_dynamic!(api.intelligence.search(query, cursor: cursor))
+          response = Structs::VirusTotalIntelligence::Response.from_dynamic!(client.intel_search(query,
+            cursor: cursor))
           responses << response
 
           break if response.meta.cursor.nil?

data/lib/mihari/analyzers/zoomeye.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "zoomeye"
-
 module Mihari
   module Analyzers
     class ZoomEye < Base
@@ -14,6 +12,15 @@ module Mihari
       # @return [String, nil]
       attr_reader :api_key
 
+      # @return [String]
+      attr_reader :query
+
+      # @return [String]
+      attr_reader :type
+
+      # @return [Integer]
+      attr_reader :interval
+
       def initialize(*args, **kwargs)
         super(*args, **kwargs)
 
@@ -48,8 +55,8 @@ module Mihari
         %w[zoomeye_api_key]
       end
 
-      def api
-        @api ||= ::ZoomEye::API.new(api_key: api_key)
+      def client
+        @client ||= Clients::ZoomEye.new(api_key: api_key)
       end
 
       #
@@ -83,9 +90,7 @@ module Mihari
       # @return [Hash, nil]
       #
       def _host_search(query, page: 1)
-        api.host.search(query, page: page)
-      rescue ::ZoomEye::Error => _e
-        nil
+        client.host_search(query, page: page)
       end
 
       #
@@ -118,9 +123,7 @@ module Mihari
       # @return [Hash, nil]
       #
       def _web_search(query, page: 1)
-        api.web.search(query, page: page)
-      rescue ::ZoomEye::Error => _e
-        nil
+        client.web_search(query, page: page)
       end
 
       #

data/lib/mihari/clients/base.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Clients
+    class Base
+      # @return [String]
+      attr_reader :base_url
+
+      # @return [Hash]
+      attr_reader :headers
+
+      #
+      # @param [String] base_url
+      # @param [Hash] headers
+      #
+      def initialize(base_url, headers: {})
+        @base_url = base_url
+        @headers = headers || {}
+      end
+
+      private
+
+      #
+      # @param [String] path
+      #
+      # @return [String]
+      #
+      def url_for(path)
+        base_url + path
+      end
+
+      #
+      # @param [String] path
+      # @param [Hashk, nil] params
+      #
+      # @return [String] <description>
+      #
+      def get(path, params: nil)
+        HTTP.get(url_for(path), headers: headers, params: params)
+      end
+
+      #
+      # @param [String] path
+      # @param [Hash, nil] json
+      #
+      # @return [String] <description>
+      #
+      def post(path, json: {})
+        HTTP.post(url_for(path), headers: headers, json: json)
+      end
+    end
+  end
+end

data/lib/mihari/clients/binaryedge.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Clients
+    class BinaryEdge < Base
+      #
+      # @param [String] base_url
+      # @param [String, nil] api_key
+      # @param [Hash] headers
+      #
+      def initialize(base_url = "https://api.binaryedge.io/v2", api_key:, headers: {})
+        raise(ArgumentError, "'api_key' argument is required") unless api_key
+
+        headers["x-key"] = api_key
+
+        super(base_url, headers: headers)
+      end
+
+      #
+      # @param [String] query String used to query our data
+      # @param [Integer] page Default 1, Maximum: 500
+      # @param [Integer, nil] only_ips If selected, only output IP addresses, ports and protocols.
+      #
+      # @return [Hash]
+      #
+      def search(query, page: 1, only_ips: nil)
+        params = {
+          query: query,
+          page: page,
+          only_ips: only_ips
+        }.compact
+
+        res = get("/query/search", params: params)
+        JSON.parse(res.body.to_s)
+      end
+    end
+  end
+end

data/lib/mihari/clients/censys.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require "base64"
+
+module Mihari
+  module Clients
+    class Censys < Base
+      #
+      # @param [String] base_url
+      # @param [String, nil] id
+      # @param [String, nil] secret
+      # @param [Hash] headers
+      #
+      def initialize(base_url = "https://search.censys.io", id:, secret:, headers: {})
+        raise(ArgumentError, "'id' argument is required") if id.nil?
+        raise(ArgumentError, "'secret' argument is required") if secret.nil?
+
+        headers["authorization"] = "Basic #{Base64.strict_encode64("#{id}:#{secret}")}"
+
+        super(base_url, headers: headers)
+      end
+
+      #
+      # Search current index.
+      #
+      # Searches the given index for all records that match the given query.
+      # For more details, see our documentation: https://search.censys.io/api/v2/docs
+      #
+      # @param [String] query the query to be executed.
+      # @params [Integer, nil] per_page the number of results to be returned for each page.
+      # @params [Integer, nil] cursor the cursor of the desired result set.
+      #
+      # @return [Structs::Censys::Response]
+      #
+      def search(query, per_page: nil, cursor: nil)
+        params = { q: query, per_page: per_page, cursor: cursor }.compact
+        res = get("/api/v2/hosts/search", params: params)
+        Structs::Censys::Response.from_dynamic! JSON.parse(res.body.to_s)
+      end
+    end
+  end
+end