mihari 8.0.2 → 8.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d3d454e8ac560ec2d1b7463dfdc686e69fc033da82e318f9479ba14f72194c49
-  data.tar.gz: d2f191e78eb2180a9062d885fd527f5e55c70fa70a5e9f8f910b15a50d57d6a2
+  metadata.gz: f8838a34fe88cc9298632b713c7fea892bcb007b19590dda803c7194f75d593d
+  data.tar.gz: 3b4de2b033ef657f99ae6e78018a912f2b19bf3098fee2e417ec7b295f40e215
 SHA512:
-  metadata.gz: 723186046c14e4532c3662280264a51fb7a5966173d7010c50d6537a8b96304b6ff7f7f24d9686d1773fe8f13ceaba09d7d33c0265e286b58c9d18f3849645b8
-  data.tar.gz: 8a9f7001f3edb2a4035178c810ff5a12cf2cf913533f853e778c8d15ef4e74edfaef4d1b9061ab406d824406e7abd9eba94adb2e45da6c9b963a5d029123d322
+  metadata.gz: 5b7a97f55459f2e123746af8964a305747122c350c75600a5da00c8b73f222d554dc817d5906df6841bb3a41fa2522988d28678e0ebec29d9451303e32888e7b
+  data.tar.gz: 22eb5f3f4ab974baadcb6f9e16c136ae18ce776ed7c7841f4e55d1cee85d1431897e3c3d00e0e44adc24daac207c7a7782007c922b6a41eefd56712c7d5e61af

data/.rubocop.yml

@@ -29,13 +29,14 @@ RSpec/SpecFilePathFormat:
 FactoryBot/SyntaxMethods:
   Enabled: false
 require:
+  - standard
+plugins:
   - rubocop-capybara
   - rubocop-factory_bot
   - rubocop-performance
   - rubocop-rake
   - rubocop-rspec
   - rubocop-yard
-  - standard
   - standard-custom
   - standard-performance
 inherit_gem:

data/Dockerfile

@@ -1,11 +1,11 @@
-FROM ruby:3.3.0-alpine3.19
+FROM ruby:3.4-alpine3.22
 
-ARG MIHARI_VERSION=0.0.0
+ARG VERSION=0.0.0
 
 RUN apk --no-cache add build-base ruby-dev libpq-dev whois && \
   echo 'gem: --no-document' >> /usr/local/etc/gemrc && \
   gem install pg && \
-  gem install mihari -v ${MIHARI_VERSION} && \
+  gem install mihari -v ${VERSION} && \
   apk del --purge build-base ruby-dev && \
   rm -rf /usr/local/bundle/cache/*
 

data/README.md

@@ -12,7 +12,6 @@ Mihari can aggregate multiple searches across multiple services in a single rule
 
 Mihari supports the following services by default.
 
-- [BinaryEdge](https://www.binaryedge.io/)
 - [Censys](http://censys.io)
 - [CIRCL passive DNS](https://www.circl.lu/services/passive-dns/) / [passive SSL](https://www.circl.lu/services/passive-ssl/)
 - [crt.sh](https://crt.sh/)

data/Rakefile

@@ -43,6 +43,10 @@ def build_swagger_doc(path)
   File.write(path, json.to_yaml)
 end
 
+def latest_tag
+  `git describe --tags --abbrev=0`.strip.sub(/^v/, "")
+end
+
 namespace :build do
   desc "Build Swagger doc"
   task :swagger, [:path] do |_t, args|

data/lib/mihari/analyzers/urlscan.rb

@@ -12,29 +12,29 @@ module Mihari
       attr_reader :api_key
 
       # @return [Array<String>]
-      attr_reader :allowed_data_types
+      attr_reader :data_types
 
       #
       # @param [String] query
       # @param [Hash, nil] options
       # @param [String, nil] api_key
-      # @param [Array<String>] allowed_data_types
+      # @param [Array<String>] data_types
       #
-      def initialize(query, options: nil, api_key: nil, allowed_data_types: SUPPORTED_DATA_TYPES)
+      def initialize(query, options: nil, api_key: nil, data_types: SUPPORTED_DATA_TYPES)
         super(query, options:)
 
         @api_key = api_key || Mihari.config.urlscan_api_key
-        @allowed_data_types = allowed_data_types
+        @data_types = data_types
 
-        return if valid_allowed_data_types?
+        return if valid_data_types?
 
-        raise ValueError, "allowed_data_types should be any of url, domain and ip."
+        raise ValueError, "data_types should be any of url, domain and ip."
       end
 
       def artifacts
         # @type [Array<Mihari::Models::Artifact>]
         artifacts = client.search_with_pagination(query, pagination_limit:).map(&:artifacts).flatten
-        artifacts.select { |artifact| allowed_data_types.include? artifact.data_type }
+        artifacts.select { |artifact| data_types.include? artifact.data_type }
       end
 
       private
@@ -52,8 +52,8 @@ module Mihari
       #
       # @return [Boolean]
       #
-      def valid_allowed_data_types?
-        allowed_data_types.all? { |type| SUPPORTED_DATA_TYPES.include? type }
+      def valid_data_types?
+        data_types.all? { |type| SUPPORTED_DATA_TYPES.include? type }
       end
     end
   end

data/lib/mihari/analyzers/zoomeye.rb

@@ -6,51 +6,39 @@ module Mihari
     # ZoomEye analyzer
     #
     class ZoomEye < Base
+      SUPPORTED_DATA_TYPES = %w[url domain ip].freeze
+
       # @return [String, nil]
       attr_reader :api_key
 
-      # @return [String]
-      attr_reader :type
+      # @return [Array<String>]
+      attr_reader :data_types
 
       #
       # @param [String] query
       # @param [Hash, nil] options
       # @param [String, nil] api_key
-      # @param [String] type
+      # @param [Array<String>] data_types
       #
-      def initialize(query, options: nil, api_key: nil, type: "host")
+      def initialize(query, options: nil, api_key: nil, data_types: SUPPORTED_DATA_TYPES)
         super(query, options:)
 
-        @type = type
         @api_key = api_key || Mihari.config.zoomeye_api_key
+        @data_types = data_types
+
+        return if valid_data_types?
+
+        raise ValueError, "data_types should be any of url, domain and ip."
       end
 
       def artifacts
-        case type
-        when "host"
-          client.host_search_with_pagination(query).map do |res|
-            convert(res)
-          end.flatten
-        when "web"
-          client.web_search_with_pagination(query).map do |res|
-            convert(res)
-          end.flatten
-        else
-          raise ValueError, "#{type} type is not supported." unless valid_type?
-        end
+        # @type [Array<Mihari::Models::Artifact>]
+        artifacts = client.search_with_pagination(query, pagination_limit:).map(&:artifacts).flatten
+        artifacts.select { |artifact| data_types.include? artifact.data_type }
       end
 
       private
 
-      #
-      # Check whether a type is valid or not
-      #
-      # @return [Boolean]
-      #
-      def valid_type?
-        %w[host web].include? type
-      end
-
       def client
         Clients::ZoomEye.new(
           api_key:,
@@ -60,23 +48,12 @@ module Mihari
       end
 
       #
-      # Convert responses into an array of String
+      # Check whether a data type is valid or not
       #
-      # @param [Hash] res
-      #
-      # @return [Array<Mihari::Models::Artifact>]
+      # @return [Boolean]
       #
-      def convert(res)
-        matches = res["matches"] || []
-        matches.map do |match|
-          data = match["ip"]
-
-          if data.is_a?(Array)
-            data.map { |d| Models::Artifact.new(data: d, metadata: match) }
-          else
-            Models::Artifact.new(data:, metadata: match)
-          end
-        end.flatten
+      def valid_data_types?
+        data_types.all? { |type| SUPPORTED_DATA_TYPES.include? type }
       end
     end
   end

data/lib/mihari/clients/zoomeye.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "base64"
+
 module Mihari
   module Clients
     #
@@ -18,7 +20,7 @@ module Mihari
       # @param [Integer, nil] timeout
       #
       def initialize(
-        base_url = "https://api.zoomeye.org",
+        base_url = "https://api.zoomeye.ai",
         api_key:,
         headers: {},
         pagination_interval: Mihari.config.pagination_interval,
@@ -38,83 +40,36 @@ module Mihari
       end
 
       #
-      # Search the Host devices
-      #
-      # @param [String] query Query string
-      # @param [Integer, nil] page The page number to paging(default:1)
-      # @param [String, nil] facets A comma-separated list of properties to get summary information on query
-      #
-      # @return [Hash]
-      #
-      def host_search(query, page: nil, facets: nil)
-        params = {
-          query:,
-          page:,
-          facets:
-        }.compact
-        get_json "/host/search", params:
-      end
-
-      #
-      # @param [String] query
-      # @param [String, nil] facets
-      # @param [Integer] pagination_limit
-      #
-      # @return [Enumerable<Hash>]
-      #
-      def host_search_with_pagination(query, facets: nil, pagination_limit: Mihari.config.pagination_limit)
-        Enumerator.new do |y|
-          (1..pagination_limit).each do |page|
-            res = host_search(query, facets:, page:)
-
-            break if res.nil?
-
-            y.yield res
-
-            total = res["total"].to_i
-            break if total <= page * PAGE_SIZE
-
-            sleep_pagination_interval
-          end
-        end
-      end
-
-      #
-      # Search the Web technologies
+      # Search
       #
       # @param [String] query Query string
       # @param [Integer, nil] page The page number to paging(default:1)
-      # @param [String, nil] facets A comma-separated list of properties to get summary information on query
       #
-      # @return [Hash]
+      # @return [Structs::ZoomEye::Response]
       #
-      def web_search(query, page: nil, facets: nil)
-        params = {
-          query:,
-          page:,
-          facets:
-        }.compact
-        get_json "/web/search", params:
+      # @param [Object, nil] facets
+      def search(query, page: nil)
+        qbase64 = Base64.urlsafe_encode64(query)
+        json = {qbase64:, page:}.compact
+        Structs::ZoomEye::Response.from_dynamic! post_json("/v2/search", json:)
       end
 
       #
       # @param [String] query
-      # @param [String, nil] facets
       # @param [Integer] pagination_limit
       #
-      # @return [Enumerable<Hash>]
+      # @return [Enumerable<Structs::ZoomEye::Response>]
       #
-      def web_search_with_pagination(query, facets: nil, pagination_limit: Mihari.config.pagination_limit)
+      def search_with_pagination(query, pagination_limit: Mihari.config.pagination_limit)
         Enumerator.new do |y|
           (1..pagination_limit).each do |page|
-            res = web_search(query, facets:, page:)
+            res = search(query, page:)
 
             break if res.nil?
 
             y.yield res
 
-            total = res["total"].to_i
-            break if total <= page * PAGE_SIZE
+            break if res.total <= page * PAGE_SIZE
 
             sleep_pagination_interval
           end

data/lib/mihari/config.rb

@@ -9,7 +9,6 @@ module Mihari
 
     attr_config(
       # analyzers, emitters & enrichers
-      binaryedge_api_key: nil,
       censys_id: nil,
       censys_secret: nil,
       circl_passive_password: nil,
@@ -55,9 +54,6 @@ module Mihari
       sentry_trace_sample_rate: 0.25
     )
 
-    # @!attribute [r] binaryedge_api_key
-    #   @return [String, nil]
-
     # @!attribute [r] censys_id
     #   @return [String, nil]
 

data/lib/mihari/schemas/analyzer.rb

@@ -10,11 +10,9 @@ module Mihari
 
       # Analyzer with API key and pagination
       [
-        Mihari::Analyzers::BinaryEdge.keys,
         Mihari::Analyzers::GreyNoise.keys,
         Mihari::Analyzers::Onyphe.keys,
         Mihari::Analyzers::Shodan.keys,
-        Mihari::Analyzers::Urlscan.keys,
         Mihari::Analyzers::Validin.keys,
         Mihari::Analyzers::VirusTotalIntelligence.keys
       ].each do |keys|
@@ -81,10 +79,17 @@ module Mihari
         optional(:options).hash(AnalyzerOptions)
       end
 
+      Urlscan = Dry::Schema.Params do
+        required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::Urlscan.keys))
+        required(:query).filled(:string)
+        optional(:data_types).filled(array[Types::NetworkDataTypes]).default(Types::NetworkDataTypes.values)
+        optional(:options).hash(AnalyzerPaginationOptions)
+      end
+
       ZoomEye = Dry::Schema.Params do
         required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::ZoomEye.keys))
         required(:query).filled(:string)
-        required(:type).value(Types::String.enum("host", "web"))
+        optional(:data_types).filled(array[Types::NetworkDataTypes]).default(Types::NetworkDataTypes.values)
         optional(:options).hash(AnalyzerPaginationOptions)
       end
 

data/lib/mihari/schemas/rule.rb

@@ -25,7 +25,7 @@ module Mihari
       optional(:emitters).array { Emitter }.default(DEFAULT_EMITTERS)
       optional(:enrichers).array { Enricher }.default(DEFAULT_ENRICHERS)
 
-      optional(:data_types).filled(array[Types::DataTypes]).default(Mihari::Types::DataTypes.values)
+      optional(:data_types).filled(array[Types::DataTypes]).default(Types::DataTypes.values)
 
       optional(:falsepositives).array { filled(:string) }.default([])
 

data/lib/mihari/structs/zoomeye.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Structs
+    module ZoomEye
+      class Datanum < Dry::Struct
+        # @!attribute [r] ip
+        #   @return [String]
+        attribute :ip, Types::String.optional
+
+        # @!attribute [r] domain
+        #   @return [String, nil]
+        attribute :domain, Types::String.optional
+
+        # @!attribute [r] url
+        #   @return [String]
+        attribute :url, Types::String.optional
+
+        # @!attribute [r] metadata
+        #   @return [Hash]
+        attribute :metadata, Types::Hash
+
+        class << self
+          #
+          # @param [Hash] d
+          #
+          def from_dynamic!(d)
+            d = Types::Hash[d]
+            new(
+              domain: d["domain"],
+              ip: d["ip"],
+              url: d["url"],
+              metadata: d
+            )
+          end
+        end
+
+        def artifacts
+          values = [url, domain, ip].compact
+          values.map { |value| Mihari::Models::Artifact.new(data: value, metadata:) }
+        end
+      end
+
+      class Response < Dry::Struct
+        # @!attribute [r] data
+        #   @return [Array<Datanum>]
+        attribute :data, Types.Array(Datanum)
+
+        # @!attribute [r] total
+        #   @return [Integer]
+        attribute :total, Types::Int
+
+        #
+        # @return [Array<Mihari::Models::Artifact>]
+        #
+        def artifacts
+          data.map(&:artifacts).flatten
+        end
+
+        class << self
+          #
+          # @param [Hash] d
+          #
+          def from_dynamic!(d)
+            d = Types::Hash[d]
+            new(
+              data: d.fetch("data").map { |x| Datanum.from_dynamic!(x) },
+              total: d.fetch("total")
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mihari/types.rb

@@ -15,7 +15,10 @@ module Mihari
     Double = Strict::Float | Strict::Integer
     DateTime = Strict::DateTime
 
-    DataTypes = Types::String.enum("hash", "ip", "domain", "url", "mail")
+    NetworkDataTypes = Types::String.enum("ip", "domain", "url")
+    DataTypes = Types::String.enum(
+      *[NetworkDataTypes.values, "hash", "mail"].flatten
+    )
 
     HTTPRequestMethods = Types::String.enum("GET", "POST")
   end

data/lib/mihari/version.rb

@@ -1,5 +1,3 @@
-# frozen_string_literal: true
-
 module Mihari
-  VERSION = "8.0.2"
+  VERSION = "8.2.0"
 end