mihari 5.4.9 → 5.5.0

data/frontend/package.json

@@ -19,7 +19,7 @@
     "@fortawesome/vue-fontawesome": "^3.0.3",
     "@vueuse/core": "^10.5.0",
     "@vueuse/router": "^10.5.0",
-    "ace-builds": "^1.29.0",
+    "ace-builds": "^1.30.0",
     "axios": "^1.5.1",
     "bulma": "^0.9.4",
     "bulma-helpers": "^0.4.3",
@@ -30,27 +30,27 @@
     "ts-dedent": "^2.2.0",
     "url-parse": "^1.5.10",
     "uuidv4": "^6.2.13",
-    "vue": "^3.3.4",
+    "vue": "^3.3.6",
     "vue-concurrency": "4.0.1",
     "vue-json-pretty": "^2.2.4",
     "vue-router": "^4.2.5",
     "vue3-ace-editor": "^2.2.3"
   },
   "devDependencies": {
-    "@redocly/cli": "1.2.0",
+    "@redocly/cli": "1.3.0",
     "@rushstack/eslint-patch": "^1.5.1",
     "@tsconfig/node20": "^20.1.2",
-    "@types/jsdom": "^21.1.3",
-    "@types/node": "^20.8.5",
-    "@types/url-parse": "^1.4.9",
-    "@typescript-eslint/eslint-plugin": "^6.7.5",
-    "@typescript-eslint/parser": "^6.7.5",
+    "@types/jsdom": "^21.1.4",
+    "@types/node": "^20.8.7",
+    "@types/url-parse": "^1.4.10",
+    "@typescript-eslint/eslint-plugin": "^6.8.0",
+    "@typescript-eslint/parser": "^6.8.0",
     "@vitejs/plugin-vue": "^4.4.0",
     "@vue/eslint-config-prettier": "^8.0.0",
     "@vue/eslint-config-typescript": "^12.0.0",
     "@vue/test-utils": "2.4.1",
     "@vue/tsconfig": "^0.4.0",
-    "eslint": "^8.51.0",
+    "eslint": "^8.52.0",
     "eslint-config-prettier": "^9.0.0",
     "eslint-plugin-prettier": "^5.0.1",
     "eslint-plugin-simple-import-sort": "^10.0.0",
@@ -60,7 +60,7 @@
     "npm-run-all": "^4.1.5",
     "prettier": "^3.0.3",
     "typescript": "~5.2.2",
-    "vite": "^4.4.11",
+    "vite": "^4.5.0",
     "vitest": "^0.34.6",
     "vue-tsc": "^1.8.19"
   }

data/lib/mihari/analyzers/base.rb

@@ -24,17 +24,17 @@ module Mihari
       end
 
       #
-      # @return [Integer, nil]
+      # @return [Integer]
       #
-      def interval
-        options[:interval]
+      def retry_interval
+        options[:retry_interval] || Mihari.config.retry_interval
       end
 
       #
-      # @return [Integer]
+      # @return [Boolean]
       #
-      def retry_interval
-        options[:retry_interval] || Mihari.config.retry_interval
+      def retry_exponential_backoff
+        options[:retry_exponential_backoff] || Mihari.config.retry_exponential_backoff
       end
 
       #
@@ -44,6 +44,13 @@ module Mihari
         options[:retry_times] || Mihari.config.retry_times
       end
 
+      #
+      # @return [Integer]
+      #
+      def pagination_interval
+        options[:pagination_interval] || Mihari.config.pagination_interval
+      end
+
       #
       # @return [Integer]
       #
@@ -81,7 +88,7 @@ module Mihari
       # @return [Array<Mihari::Artifact>]
       #
       def normalized_artifacts
-        retry_on_error(times: retry_times, interval: retry_interval) do
+        retry_on_error(times: retry_times, interval: retry_interval, exponential_backoff: retry_exponential_backoff) do
           artifacts.compact.sort.map do |artifact|
             # No need to set data_type manually
             # It is set automatically in #initialize
@@ -104,7 +111,7 @@ module Mihari
         self.class.to_s.split("::").last
       end
 
-      alias source class_name
+      alias_method :source, :class_name
 
       class << self
         #

data/lib/mihari/analyzers/binaryedge.rb

@@ -32,7 +32,11 @@ module Mihari
       # @return [Mihari::Clients::BinaryEdge]
       #
       def client
-        @client ||= Clients::BinaryEdge.new(api_key: api_key, interval: interval, timeout: timeout)
+        @client ||= Clients::BinaryEdge.new(
+          api_key: api_key,
+          pagination_interval: pagination_interval,
+          timeout: timeout
+        )
       end
     end
   end

data/lib/mihari/analyzers/censys.rb

@@ -52,7 +52,12 @@ module Mihari
       # @return [Mihari::Clients::Censys]
       #
       def client
-        @client ||= Clients::Censys.new(id: id, secret: secret, interval: interval, timeout: timeout)
+        @client ||= Clients::Censys.new(
+          id: id,
+          secret: secret,
+          pagination_interval: pagination_interval,
+          timeout: timeout
+        )
       end
 
       #

data/lib/mihari/analyzers/greynoise.rb

@@ -31,7 +31,11 @@ module Mihari
       private
 
       def client
-        @client ||= Clients::GreyNoise.new(api_key: api_key, interval: interval, timeout: timeout)
+        @client ||= Clients::GreyNoise.new(
+          api_key: api_key,
+          pagination_interval: pagination_interval,
+          timeout: timeout
+        )
       end
     end
   end

data/lib/mihari/analyzers/hunterhow.rb

@@ -46,7 +46,11 @@ module Mihari
       private
 
       def client
-        @client ||= Clients::HunterHow.new(api_key: api_key, interval: interval, timeout: timeout)
+        @client ||= Clients::HunterHow.new(
+          api_key: api_key,
+          pagination_interval: pagination_interval,
+          timeout: timeout
+        )
       end
     end
   end

data/lib/mihari/analyzers/onyphe.rb

@@ -33,7 +33,11 @@ module Mihari
       private
 
       def client
-        @client ||= Clients::Onyphe.new(api_key: api_key, interval: interval, timeout: timeout)
+        @client ||= Clients::Onyphe.new(
+          api_key: api_key,
+          pagination_interval: pagination_interval,
+          timeout: timeout
+        )
       end
     end
   end

data/lib/mihari/analyzers/rule.rb

@@ -35,17 +35,24 @@ module Mihari
       "webhook" => Emitters::Webhook
     }.freeze
 
+    ENRICHER_TO_CLASS = {
+      "whois" => Enrichers::Whois,
+      "ipinfo" => Enrichers::IPInfo,
+      "shodan" => Enrichers::Shodan,
+      "google_public_dns" => Enrichers::GooglePublicDNS
+    }.freeze
+
     class Rule
       include Mixins::FalsePositive
 
-      # @return [Mihari::Services::Rule]
+      # @return [Mihari::Services::RuleProxy]
       attr_reader :rule
 
       # @return [Time]
       attr_reader :base_time
 
       #
-      # @param [Mihari::Services::Rule] rule
+      # @param [Mihari::Services::RuleProxy] rule
       #
       def initialize(rule)
         @rule = rule
@@ -106,7 +113,7 @@ module Mihari
       #
       def enriched_artifacts
         @enriched_artifacts ||= Parallel.map(unique_artifacts) do |artifact|
-          rule.enrichers.each { |enricher| artifact.enrich_by_enricher enricher[:enricher] }
+          enrichers.each { |enricher| artifact.enrich_by_enricher enricher }
           artifact
         end
       end
@@ -193,18 +200,18 @@ module Mihari
         raise ArgumentError, "#{emitter_name} is not supported"
       end
 
-      #
-      # Deep copied emitters
       #
       # @return [Array<Mihari::Emitters::Base>]
       #
       def emitters
         rule.emitters.map(&:deep_dup).map do |params|
           name = params[:emitter]
-          params.delete(:emitter)
+          options = params[:options]
+
+          %i[emitter options].each { |key| params.delete key }
 
           klass = get_emitter_class(name)
-          klass.new(artifacts: enriched_artifacts, rule: rule, **params)
+          klass.new(artifacts: enriched_artifacts, rule: rule, options: options, **params)
         end
       end
 
@@ -215,6 +222,35 @@ module Mihari
         emitters.select(&:valid?)
       end
 
+      #
+      # Get enricher class
+      #
+      # @param [String] enricher_name
+      #
+      # @return [Class<Mihari::Enrichers::Base>] enricher class
+      #
+      def get_enricher_class(enricher_name)
+        enricher = ENRICHER_TO_CLASS[enricher_name]
+        return enricher if enricher
+
+        raise ArgumentError, "#{enricher_name} is not supported"
+      end
+
+      #
+      # @return [Array<Mihari::Enrichers::Base>] enrichers
+      #
+      def enrichers
+        @enrichers ||= rule.enrichers.map(&:deep_dup).map do |params|
+          name = params[:enricher]
+          options = params[:options]
+
+          %i[enricher options].each { |key| params.delete key }
+
+          klass = get_enricher_class(name)
+          klass.new(options: options, **params)
+        end
+      end
+
       #
       # Validate configuration of analyzers
       #

data/lib/mihari/analyzers/shodan.rb

@@ -34,7 +34,11 @@ module Mihari
       # @return [Clients::Shodan]
       #
       def client
-        @client ||= Clients::Shodan.new(api_key: api_key, interval: interval, timeout: timeout)
+        @client ||= Clients::Shodan.new(
+          api_key: api_key,
+          pagination_interval: pagination_interval,
+          timeout: timeout
+        )
       end
     end
   end

data/lib/mihari/analyzers/urlscan.rb

@@ -44,7 +44,11 @@ module Mihari
       private
 
       def client
-        @client ||= Clients::UrlScan.new(api_key: api_key, interval: interval, timeout: timeout)
+        @client ||= Clients::UrlScan.new(
+          api_key: api_key,
+          pagination_interval: pagination_interval,
+          timeout: timeout
+        )
       end
 
       #

data/lib/mihari/analyzers/virustotal_intelligence.rb

@@ -33,7 +33,11 @@ module Mihari
       # @return [::VirusTotal::API]
       #
       def client
-        @client = Clients::VirusTotal.new(api_key: api_key, interval: interval, timeout: timeout)
+        @client = Clients::VirusTotal.new(
+          api_key: api_key,
+          pagination_interval: pagination_interval,
+          timeout: timeout
+        )
       end
     end
   end

data/lib/mihari/analyzers/zoomeye.rb

@@ -53,7 +53,11 @@ module Mihari
       end
 
       def client
-        @client ||= Clients::ZoomEye.new(api_key: api_key, interval: interval, timeout: timeout)
+        @client ||= Clients::ZoomEye.new(
+          api_key: api_key,
+          pagination_interval: pagination_interval,
+          timeout: timeout
+        )
       end
 
       #

data/lib/mihari/clients/base.rb

@@ -9,8 +9,8 @@ module Mihari
       # @return [Hash]
       attr_reader :headers
 
-      # @return [Integer, nil]
-      attr_reader :interval
+      # @return [Integer]
+      attr_reader :pagination_interval
 
       # @return [Integer, nil]
       attr_reader :timeout
@@ -18,20 +18,20 @@ module Mihari
       #
       # @param [String] base_url
       # @param [Hash] headers
-      # @param [Integer, nil] interval
+      # @param [Integer] interval
       # @param [Integer, nil] timeout
       #
-      def initialize(base_url, headers: {}, interval: nil, timeout: nil)
+      def initialize(base_url, headers: {}, pagination_interval: 0, timeout: nil)
         @base_url = base_url
         @headers = headers || {}
-        @interval = interval
+        @pagination_interval = pagination_interval
         @timeout = timeout
       end
 
       private
 
-      def sleep_interval
-        sleep(interval) if interval
+      def sleep_pagination_interval
+        sleep pagination_interval
       end
 
       #

data/lib/mihari/clients/binaryedge.rb

@@ -7,15 +7,21 @@ module Mihari
       # @param [String] base_url
       # @param [String, nil] api_key
       # @param [Hash] headers
-      # @param [Integer, nil] interval
+      # @param [Integer] pagnation_interval
       # @param [Integer, nil] timeout
       #
-      def initialize(base_url = "https://api.binaryedge.io/v2", api_key:, headers: {}, interval: nil, timeout: nil)
+      def initialize(
+        base_url = "https://api.binaryedge.io/v2",
+        api_key:,
+        headers: {},
+        pagination_interval: 0,
+        timeout: nil
+      )
         raise(ArgumentError, "'api_key' argument is required") unless api_key
 
         headers["x-key"] = api_key
 
-        super(base_url, headers: headers, interval: interval, timeout: timeout)
+        super(base_url, headers: headers, pagination_interval: pagination_interval, timeout: timeout)
       end
 
       #
@@ -52,7 +58,7 @@ module Mihari
 
             break if res.events.length < res.pagesize
 
-            sleep_interval
+            sleep_pagination_interval
           end
         end
       end

data/lib/mihari/clients/censys.rb

@@ -10,16 +10,23 @@ module Mihari
       # @param [String, nil] id
       # @param [String, nil] secret
       # @param [Hash] headers
-      # @param [Integer, nil] interval
+      # @param [Integer] pagination_interval
       # @param [Integer, nil] timeout
       #
-      def initialize(base_url = "https://search.censys.io", id:, secret:, headers: {}, interval: nil, timeout: nil)
+      def initialize(
+        base_url = "https://search.censys.io",
+        id:,
+        secret:,
+        headers: {},
+        pagination_interval: 0,
+        timeout: nil
+      )
         raise(ArgumentError, "'id' argument is required") if id.nil?
         raise(ArgumentError, "'secret' argument is required") if secret.nil?
 
         headers["authorization"] = "Basic #{Base64.strict_encode64("#{id}:#{secret}")}"
 
-        super(base_url, headers: headers, interval: interval, timeout: timeout)
+        super(base_url, headers: headers, pagination_interval: pagination_interval, timeout: timeout)
       end
 
       #
@@ -64,7 +71,7 @@ module Mihari
             # So it needs to check both cases
             break if cursor.nil? || cursor.empty?
 
-            sleep_interval
+            sleep_pagination_interval
           end
         end
       end

data/lib/mihari/clients/greynoise.rb

@@ -9,14 +9,20 @@ module Mihari
       # @param [String] base_url
       # @param [String, nil] api_key
       # @param [Hash] headers
-      # @param [Integer, nil] interval
+      # @param [Integer] pagination_interval
       # @param [Integer, nil] timeout
       #
-      def initialize(base_url = "https://api.greynoise.io", api_key:, headers: {}, interval: nil, timeout: nil)
+      def initialize(
+        base_url = "https://api.greynoise.io",
+        api_key:,
+        headers: {},
+        pagination_interval: 0,
+        timeout: nil
+      )
         raise(ArgumentError, "'api_key' argument is required") unless api_key
 
         headers["key"] = api_key
-        super(base_url, headers: headers, interval: interval, timeout: timeout)
+        super(base_url, headers: headers, pagination_interval: pagination_interval, timeout: timeout)
       end
 
       #
@@ -53,7 +59,7 @@ module Mihari
             scroll = res.scroll
             break if scroll.nil?
 
-            sleep_interval
+            sleep_pagination_interval
           end
         end
       end

data/lib/mihari/clients/hunterhow.rb

@@ -14,13 +14,19 @@ module Mihari
       # @param [String] base_url
       # @param [String, nil] api_key
       # @param [Hash] headers
-      # @param [Integer, nil] interval
+      # @param [Integer] pagination_interval
       # @param [Integer, nil] timeout
       #
-      def initialize(base_url = "https://api.hunter.how/", api_key:, headers: {}, interval: nil, timeout: nil)
+      def initialize(
+        base_url = "https://api.hunter.how/",
+        api_key:,
+        headers: {},
+        pagination_interval: 0,
+        timeout: nil
+      )
         raise(ArgumentError, "'api_key' argument is required") unless api_key
 
-        super(base_url, headers: headers, interval: interval, timeout: timeout)
+        super(base_url, headers: headers, pagination_interval: pagination_interval, timeout: timeout)
 
         @api_key = api_key
       end
@@ -77,7 +83,7 @@ module Mihari
 
             break if res.data.list.length < page_size
 
-            sleep_interval
+            sleep_pagination_interval
           end
         end
       end

data/lib/mihari/clients/misp.rb

@@ -7,14 +7,15 @@ module Mihari
       # @param [String] base_url
       # @param [String, nil] api_key
       # @param [Hash] headers
+      # @param [Integer, nil] timeout
       #
-      def initialize(base_url, api_key:, headers: {})
+      def initialize(base_url, api_key:, headers: {}, timeout: nil)
         raise(ArgumentError, "'api_key' argument is required") unless api_key
 
         headers["authorization"] = api_key
         headers["accept"] = "application/json"
 
-        super(base_url, headers: headers)
+        super(base_url, headers: headers, timeout: timeout)
       end
 
       #

data/lib/mihari/clients/onyphe.rb

@@ -12,13 +12,19 @@ module Mihari
       # @param [String] base_url
       # @param [String, nil] api_key
       # @param [Hash] headers
-      # @param [Integer, nil] interval
+      # @param [Integer] pagination_interval
       # @param [Integer, nil] timeout
       #
-      def initialize(base_url = "https://www.onyphe.io", api_key:, headers: {}, interval: nil, timeout: nil)
+      def initialize(
+        base_url = "https://www.onyphe.io",
+        api_key:,
+        headers: {},
+        pagination_interval: 0,
+        timeout: nil
+      )
         raise(ArgumentError, "'api_key' argument is required") if api_key.nil?
 
-        super(base_url, headers: headers, interval: interval, timeout: timeout)
+        super(base_url, headers: headers, pagination_interval: pagination_interval, timeout: timeout)
 
         @api_key = api_key
       end
@@ -50,7 +56,7 @@ module Mihari
 
             break if res.total <= page * PAGE_SIZE
 
-            sleep_interval
+            sleep_pagination_interval
           end
         end
       end

data/lib/mihari/clients/shodan.rb

@@ -12,13 +12,19 @@ module Mihari
       # @param [String] base_url
       # @param [String, nil] api_key
       # @param [Hash] headers
-      # @param [Integer, nil] interval
+      # @param [Integer] pagination_interval
       # @param [Integer, nil] timeout
       #
-      def initialize(base_url = "https://api.shodan.io", api_key:, headers: {}, interval: nil, timeout: nil)
+      def initialize(
+        base_url = "https://api.shodan.io",
+        api_key:,
+        headers: {},
+        pagination_interval: 0,
+        timeout: nil
+      )
         raise(ArgumentError, "'api_key' argument is required") unless api_key
 
-        super(base_url, headers: headers, interval: interval, timeout: timeout)
+        super(base_url, headers: headers, pagination_interval: pagination_interval, timeout: timeout)
 
         @api_key = api_key
       end
@@ -57,7 +63,7 @@ module Mihari
 
             break if res.total <= page * PAGE_SIZE
 
-            sleep_interval
+            sleep_pagination_interval
           rescue JSON::ParserError
             # ignore JSON::ParserError
             # ref. https://github.com/ninoseki/mihari/issues/197

data/lib/mihari/clients/the_hive.rb

@@ -8,14 +8,15 @@ module Mihari
       # @param [String, nil] api_key
       # @param [String, nil] api_version
       # @param [Hash] headers
+      # @param [Integer, nil] timeout
       #
-      def initialize(base_url, api_key:, api_version:, headers: {})
+      def initialize(base_url, api_key:, api_version:, headers: {}, timeout: nil)
         raise(ArgumentError, "'api_key' argument is required") unless api_key
 
         base_url += "/#{api_version}" unless api_version.nil?
         headers["authorization"] = "Bearer #{api_key}"
 
-        super(base_url, headers: headers)
+        super(base_url, headers: headers, timeout: timeout)
       end
 
       #

data/lib/mihari/clients/urlscan.rb

@@ -10,12 +10,18 @@ module Mihari
       # @param [Interval, nil] interval
       # @param [Interval, nil] timeout
       #
-      def initialize(base_url = "https://urlscan.io", api_key:, headers: {}, interval: nil, timeout: nil)
+      def initialize(
+        base_url = "https://urlscan.io",
+        api_key:,
+        headers: {},
+        pagination_interval: 0,
+        timeout: nil
+      )
         raise(ArgumentError, "'api_key' argument is required") if api_key.nil?
 
         headers["api-key"] = api_key
 
-        super(base_url, headers: headers, interval: interval, timeout: timeout)
+        super(base_url, headers: headers, pagination_interval: pagination_interval, timeout: timeout)
       end
 
       #
@@ -51,7 +57,7 @@ module Mihari
 
             search_after = res.results.last.sort.join(",")
 
-            sleep_interval
+            sleep_pagination_interval
           end
         end
       end

data/lib/mihari/clients/virustotal.rb

@@ -7,15 +7,21 @@ module Mihari
       # @param [String] base_url
       # @param [String, nil] api_key
       # @param [Hash] headers
-      # @param [Integer, nil] interval
+      # @param [Integer] pagination_interval
       # @param [Integer, nil] timeout
       #
-      def initialize(base_url = "https://www.virustotal.com", api_key:, headers: {}, interval: nil, timeout: nil)
+      def initialize(
+        base_url = "https://www.virustotal.com",
+        api_key:,
+        headers: {},
+        pagination_interval: 0,
+        timeout: nil
+      )
         raise(ArgumentError, "'api_key' argument is required") if api_key.nil?
 
         headers["x-apikey"] = api_key
 
-        super(base_url, headers: headers, interval: interval, timeout: timeout)
+        super(base_url, headers: headers, pagination_interval: pagination_interval, timeout: timeout)
       end
 
       #
@@ -66,7 +72,7 @@ module Mihari
             cursor = res.meta.cursor
             break if cursor.nil?
 
-            sleep_interval
+            sleep_pagination_interval
           end
         end
       end