mihari 8.0.2 → 8.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d3d454e8ac560ec2d1b7463dfdc686e69fc033da82e318f9479ba14f72194c49
-  data.tar.gz: d2f191e78eb2180a9062d885fd527f5e55c70fa70a5e9f8f910b15a50d57d6a2
+  metadata.gz: d0f4a9365970c476890029a56b880a81383fcb3c8c900e123542ec8c0bd52756
+  data.tar.gz: 323805a72e47dacb248a3f4737ebc5f1890350875cc881a4e52619a5e1d09509
 SHA512:
-  metadata.gz: 723186046c14e4532c3662280264a51fb7a5966173d7010c50d6537a8b96304b6ff7f7f24d9686d1773fe8f13ceaba09d7d33c0265e286b58c9d18f3849645b8
-  data.tar.gz: 8a9f7001f3edb2a4035178c810ff5a12cf2cf913533f853e778c8d15ef4e74edfaef4d1b9061ab406d824406e7abd9eba94adb2e45da6c9b963a5d029123d322
+  metadata.gz: ec6bb62f4d03438cc83052ca4b53950bdd13e22abab9435e92aeb993b6809c139a53f02b299f2706512aa67f72370086f2433573165b256dab1973331929b7ac
+  data.tar.gz: 976fe873b455de92f917bde4de403531e104d3378820364a4f7ddf9274b13298e474891c774df6008c1d4e8099be391f73723cc8ea9f73839a9087fc66eb915d

data/lib/mihari/analyzers/urlscan.rb

@@ -12,29 +12,29 @@ module Mihari
       attr_reader :api_key
 
       # @return [Array<String>]
-      attr_reader :allowed_data_types
+      attr_reader :data_types
 
       #
       # @param [String] query
       # @param [Hash, nil] options
       # @param [String, nil] api_key
-      # @param [Array<String>] allowed_data_types
+      # @param [Array<String>] data_types
       #
-      def initialize(query, options: nil, api_key: nil, allowed_data_types: SUPPORTED_DATA_TYPES)
+      def initialize(query, options: nil, api_key: nil, data_types: SUPPORTED_DATA_TYPES)
         super(query, options:)
 
         @api_key = api_key || Mihari.config.urlscan_api_key
-        @allowed_data_types = allowed_data_types
+        @data_types = data_types
 
-        return if valid_allowed_data_types?
+        return if valid_data_types?
 
-        raise ValueError, "allowed_data_types should be any of url, domain and ip."
+        raise ValueError, "data_types should be any of url, domain and ip."
       end
 
       def artifacts
         # @type [Array<Mihari::Models::Artifact>]
         artifacts = client.search_with_pagination(query, pagination_limit:).map(&:artifacts).flatten
-        artifacts.select { |artifact| allowed_data_types.include? artifact.data_type }
+        artifacts.select { |artifact| data_types.include? artifact.data_type }
       end
 
       private
@@ -52,8 +52,8 @@ module Mihari
       #
       # @return [Boolean]
       #
-      def valid_allowed_data_types?
-        allowed_data_types.all? { |type| SUPPORTED_DATA_TYPES.include? type }
+      def valid_data_types?
+        data_types.all? { |type| SUPPORTED_DATA_TYPES.include? type }
       end
     end
   end

data/lib/mihari/analyzers/zoomeye.rb

@@ -6,51 +6,39 @@ module Mihari
     # ZoomEye analyzer
     #
     class ZoomEye < Base
+      SUPPORTED_DATA_TYPES = %w[url domain ip].freeze
+
       # @return [String, nil]
       attr_reader :api_key
 
-      # @return [String]
-      attr_reader :type
+      # @return [Array<String>]
+      attr_reader :data_types
 
       #
       # @param [String] query
       # @param [Hash, nil] options
       # @param [String, nil] api_key
-      # @param [String] type
+      # @param [Array<String>] data_types
       #
-      def initialize(query, options: nil, api_key: nil, type: "host")
+      def initialize(query, options: nil, api_key: nil, data_types: SUPPORTED_DATA_TYPES)
         super(query, options:)
 
-        @type = type
         @api_key = api_key || Mihari.config.zoomeye_api_key
+        @data_types = data_types
+
+        return if valid_data_types?
+
+        raise ValueError, "data_types should be any of url, domain and ip."
       end
 
       def artifacts
-        case type
-        when "host"
-          client.host_search_with_pagination(query).map do |res|
-            convert(res)
-          end.flatten
-        when "web"
-          client.web_search_with_pagination(query).map do |res|
-            convert(res)
-          end.flatten
-        else
-          raise ValueError, "#{type} type is not supported." unless valid_type?
-        end
+        # @type [Array<Mihari::Models::Artifact>]
+        artifacts = client.search_with_pagination(query, pagination_limit:).map(&:artifacts).flatten
+        artifacts.select { |artifact| data_types.include? artifact.data_type }
       end
 
       private
 
-      #
-      # Check whether a type is valid or not
-      #
-      # @return [Boolean]
-      #
-      def valid_type?
-        %w[host web].include? type
-      end
-
       def client
         Clients::ZoomEye.new(
           api_key:,
@@ -60,23 +48,12 @@ module Mihari
       end
 
       #
-      # Convert responses into an array of String
+      # Check whether a data type is valid or not
       #
-      # @param [Hash] res
-      #
-      # @return [Array<Mihari::Models::Artifact>]
+      # @return [Boolean]
       #
-      def convert(res)
-        matches = res["matches"] || []
-        matches.map do |match|
-          data = match["ip"]
-
-          if data.is_a?(Array)
-            data.map { |d| Models::Artifact.new(data: d, metadata: match) }
-          else
-            Models::Artifact.new(data:, metadata: match)
-          end
-        end.flatten
+      def valid_data_types?
+        data_types.all? { |type| SUPPORTED_DATA_TYPES.include? type }
       end
     end
   end

data/lib/mihari/clients/zoomeye.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "base64"
+
 module Mihari
   module Clients
     #
@@ -18,7 +20,7 @@ module Mihari
       # @param [Integer, nil] timeout
       #
       def initialize(
-        base_url = "https://api.zoomeye.org",
+        base_url = "https://api.zoomeye.ai",
         api_key:,
         headers: {},
         pagination_interval: Mihari.config.pagination_interval,
@@ -38,83 +40,36 @@ module Mihari
       end
 
       #
-      # Search the Host devices
-      #
-      # @param [String] query Query string
-      # @param [Integer, nil] page The page number to paging(default:1)
-      # @param [String, nil] facets A comma-separated list of properties to get summary information on query
-      #
-      # @return [Hash]
-      #
-      def host_search(query, page: nil, facets: nil)
-        params = {
-          query:,
-          page:,
-          facets:
-        }.compact
-        get_json "/host/search", params:
-      end
-
-      #
-      # @param [String] query
-      # @param [String, nil] facets
-      # @param [Integer] pagination_limit
-      #
-      # @return [Enumerable<Hash>]
-      #
-      def host_search_with_pagination(query, facets: nil, pagination_limit: Mihari.config.pagination_limit)
-        Enumerator.new do |y|
-          (1..pagination_limit).each do |page|
-            res = host_search(query, facets:, page:)
-
-            break if res.nil?
-
-            y.yield res
-
-            total = res["total"].to_i
-            break if total <= page * PAGE_SIZE
-
-            sleep_pagination_interval
-          end
-        end
-      end
-
-      #
-      # Search the Web technologies
+      # Search
       #
       # @param [String] query Query string
       # @param [Integer, nil] page The page number to paging(default:1)
-      # @param [String, nil] facets A comma-separated list of properties to get summary information on query
       #
-      # @return [Hash]
+      # @return [Structs::ZoomEye::Response]
       #
-      def web_search(query, page: nil, facets: nil)
-        params = {
-          query:,
-          page:,
-          facets:
-        }.compact
-        get_json "/web/search", params:
+      # @param [Object, nil] facets
+      def search(query, page: nil)
+        qbase64 = Base64.urlsafe_encode64(query)
+        json = {qbase64:, page:}.compact
+        Structs::ZoomEye::Response.from_dynamic! post_json("/v2/search", json:)
       end
 
       #
       # @param [String] query
-      # @param [String, nil] facets
       # @param [Integer] pagination_limit
       #
-      # @return [Enumerable<Hash>]
+      # @return [Enumerable<Structs::ZoomEye::Response>]
       #
-      def web_search_with_pagination(query, facets: nil, pagination_limit: Mihari.config.pagination_limit)
+      def search_with_pagination(query, pagination_limit: Mihari.config.pagination_limit)
         Enumerator.new do |y|
           (1..pagination_limit).each do |page|
-            res = web_search(query, facets:, page:)
+            res = search(query, page:)
 
             break if res.nil?
 
             y.yield res
 
-            total = res["total"].to_i
-            break if total <= page * PAGE_SIZE
+            break if res.total <= page * PAGE_SIZE
 
             sleep_pagination_interval
           end

data/lib/mihari/schemas/analyzer.rb

@@ -14,7 +14,6 @@ module Mihari
         Mihari::Analyzers::GreyNoise.keys,
         Mihari::Analyzers::Onyphe.keys,
         Mihari::Analyzers::Shodan.keys,
-        Mihari::Analyzers::Urlscan.keys,
         Mihari::Analyzers::Validin.keys,
         Mihari::Analyzers::VirusTotalIntelligence.keys
       ].each do |keys|
@@ -81,10 +80,17 @@ module Mihari
         optional(:options).hash(AnalyzerOptions)
       end
 
+      Urlscan = Dry::Schema.Params do
+        required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::Urlscan.keys))
+        required(:query).filled(:string)
+        optional(:data_types).filled(array[Types::NetworkDataTypes]).default(Types::NetworkDataTypes.values)
+        optional(:options).hash(AnalyzerPaginationOptions)
+      end
+
       ZoomEye = Dry::Schema.Params do
         required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::ZoomEye.keys))
         required(:query).filled(:string)
-        required(:type).value(Types::String.enum("host", "web"))
+        optional(:data_types).filled(array[Types::NetworkDataTypes]).default(Types::NetworkDataTypes.values)
         optional(:options).hash(AnalyzerPaginationOptions)
       end
 

data/lib/mihari/schemas/rule.rb

@@ -25,7 +25,7 @@ module Mihari
       optional(:emitters).array { Emitter }.default(DEFAULT_EMITTERS)
       optional(:enrichers).array { Enricher }.default(DEFAULT_ENRICHERS)
 
-      optional(:data_types).filled(array[Types::DataTypes]).default(Mihari::Types::DataTypes.values)
+      optional(:data_types).filled(array[Types::DataTypes]).default(Types::DataTypes.values)
 
       optional(:falsepositives).array { filled(:string) }.default([])
 

data/lib/mihari/structs/zoomeye.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Structs
+    module ZoomEye
+      class Datanum < Dry::Struct
+        # @!attribute [r] ip
+        #   @return [String]
+        attribute :ip, Types::String.optional
+
+        # @!attribute [r] domain
+        #   @return [String, nil]
+        attribute :domain, Types::String.optional
+
+        # @!attribute [r] url
+        #   @return [String]
+        attribute :url, Types::String.optional
+
+        # @!attribute [r] metadata
+        #   @return [Hash]
+        attribute :metadata, Types::Hash
+
+        class << self
+          #
+          # @param [Hash] d
+          #
+          def from_dynamic!(d)
+            d = Types::Hash[d]
+            new(
+              domain: d["domain"],
+              ip: d["ip"],
+              url: d["url"],
+              metadata: d
+            )
+          end
+        end
+
+        def artifacts
+          values = [url, domain, ip].compact
+          values.map { |value| Mihari::Models::Artifact.new(data: value, metadata:) }
+        end
+      end
+
+      class Response < Dry::Struct
+        # @!attribute [r] data
+        #   @return [Array<Datanum>]
+        attribute :data, Types.Array(Datanum)
+
+        # @!attribute [r] total
+        #   @return [Integer]
+        attribute :total, Types::Int
+
+        #
+        # @return [Array<Mihari::Models::Artifact>]
+        #
+        def artifacts
+          data.map(&:artifacts).flatten
+        end
+
+        class << self
+          #
+          # @param [Hash] d
+          #
+          def from_dynamic!(d)
+            d = Types::Hash[d]
+            new(
+              data: d.fetch("data").map { |x| Datanum.from_dynamic!(x) },
+              total: d.fetch("total")
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mihari/types.rb

@@ -15,7 +15,10 @@ module Mihari
     Double = Strict::Float | Strict::Integer
     DateTime = Strict::DateTime
 
-    DataTypes = Types::String.enum("hash", "ip", "domain", "url", "mail")
+    NetworkDataTypes = Types::String.enum("ip", "domain", "url")
+    DataTypes = Types::String.enum(
+      *[NetworkDataTypes.values, "hash", "mail"].flatten
+    )
 
     HTTPRequestMethods = Types::String.enum("GET", "POST")
   end

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "8.0.2"
+  VERSION = "8.1.0"
 end