mihari 5.4.2 → 5.4.4

data/frontend/package.json

@@ -17,15 +17,15 @@
     "@fortawesome/fontawesome-svg-core": "^6.4.2",
     "@fortawesome/free-solid-svg-icons": "^6.4.2",
     "@fortawesome/vue-fontawesome": "^3.0.3",
-    "@vueuse/core": "^10.3.0",
-    "@vueuse/router": "^10.3.0",
-    "ace-builds": "^1.24.1",
-    "axios": "^1.4.0",
+    "@vueuse/core": "^10.4.1",
+    "@vueuse/router": "^10.4.1",
+    "ace-builds": "^1.28.0",
+    "axios": "^1.5.0",
     "bulma": "^0.9.4",
     "bulma-helpers": "^0.4.3",
-    "dayjs": "^1.11.9",
+    "dayjs": "^1.11.10",
     "font-awesome-animation": "^1.1.1",
-    "js-sha256": "^0.9.0",
+    "js-sha256": "^0.10.1",
     "truncate": "^3.0.0",
     "ts-dedent": "^2.2.0",
     "url-parse": "^1.5.10",
@@ -33,24 +33,24 @@
     "vue": "^3.3.4",
     "vue-concurrency": "4.0.1",
     "vue-json-pretty": "^2.2.4",
-    "vue-router": "^4.2.4",
+    "vue-router": "^4.2.5",
     "vue3-ace-editor": "^2.2.3"
   },
   "devDependencies": {
-    "@redocly/cli": "1.0.2",
-    "@rushstack/eslint-patch": "^1.3.3",
-    "@tsconfig/node20": "^20.1.1",
-    "@types/jsdom": "^21.1.1",
-    "@types/node": "^20.5.1",
-    "@types/url-parse": "^1.4.8",
-    "@typescript-eslint/eslint-plugin": "^6.4.0",
-    "@typescript-eslint/parser": "^6.4.0",
-    "@vitejs/plugin-vue": "^4.3.1",
+    "@redocly/cli": "1.2.0",
+    "@rushstack/eslint-patch": "^1.4.0",
+    "@tsconfig/node20": "^20.1.2",
+    "@types/jsdom": "^21.1.3",
+    "@types/node": "^20.6.4",
+    "@types/url-parse": "^1.4.9",
+    "@typescript-eslint/eslint-plugin": "^6.7.2",
+    "@typescript-eslint/parser": "^6.7.2",
+    "@vitejs/plugin-vue": "^4.3.4",
     "@vue/eslint-config-prettier": "^8.0.0",
-    "@vue/eslint-config-typescript": "^11.0.3",
+    "@vue/eslint-config-typescript": "^12.0.0",
     "@vue/test-utils": "2.4.1",
     "@vue/tsconfig": "^0.4.0",
-    "eslint": "^8.47.0",
+    "eslint": "^8.50.0",
     "eslint-config-prettier": "^9.0.0",
     "eslint-plugin-prettier": "^5.0.0",
     "eslint-plugin-simple-import-sort": "^10.0.0",
@@ -58,10 +58,10 @@
     "husky": "^8.0.3",
     "jsdom": "^22.1.0",
     "npm-run-all": "^4.1.5",
-    "prettier": "^3.0.2",
-    "typescript": "~5.1.6",
+    "prettier": "^3.0.3",
+    "typescript": "~5.2.2",
     "vite": "^4.4.9",
-    "vitest": "^0.34.2",
-    "vue-tsc": "^1.8.8"
+    "vitest": "^0.34.5",
+    "vue-tsc": "^1.8.13"
   }
 }

data/lib/mihari/analyzers/base.rb

@@ -3,6 +3,8 @@
 module Mihari
   module Analyzers
     class Base
+      include Dry::Monads[:result, :try]
+
       include Mixins::Configurable
       include Mixins::Retriable
 
@@ -25,28 +27,38 @@ module Mihari
       # @return [Integer, nil]
       #
       def interval
-        @interval ||= options[:interval]
+        options[:interval]
       end
 
       #
       # @return [Integer]
       #
       def retry_interval
-        @retry_interval ||= options[:retry_interval] || Mihari.config.retry_interval
+        options[:retry_interval] || Mihari.config.retry_interval
       end
 
       #
       # @return [Integer]
       #
       def retry_times
-        @retry_times ||= options[:retry_times] || Mihari.config.retry_times
+        options[:retry_times] || Mihari.config.retry_times
       end
 
       #
       # @return [Integer]
       #
       def pagination_limit
-        @pagination_limit ||= options[:pagination_limit] || Mihari.config.pagination_limit
+        options[:pagination_limit] || Mihari.config.pagination_limit
+      end
+
+      #
+      # @return [Boolean]
+      #
+      def ignore_error?
+        ignore_error = options[:ignore_error]
+        return ignore_error unless ignore_error.nil?
+
+        Mihari.config.ignore_error
       end
 
       # @return [Array<String>, Array<Mihari::Artifact>]
@@ -63,7 +75,7 @@ module Mihari
       #
       def normalized_artifacts
         retry_on_error(times: retry_times, interval: retry_interval) do
-          @normalized_artifacts ||= artifacts.compact.sort.map do |artifact|
+          artifacts.compact.sort.map do |artifact|
             # No need to set data_type manually
             # It is set automatically in #initialize
             artifact = artifact.is_a?(Artifact) ? artifact : Artifact.new(data: artifact)
@@ -73,6 +85,13 @@ module Mihari
         end
       end
 
+      #
+      # @return [Dry::Monads::Result::Success<Array<Mihari::Artifact>>, Dry::Monads::Result::Failure]
+      #
+      def result
+        Try[StandardError] { normalized_artifacts }.to_result
+      end
+
       # @return [String]
       def class_name
         self.class.to_s.split("::").last
@@ -80,12 +99,6 @@ module Mihari
 
       alias_method :source, :class_name
 
-      private
-
-      def sleep_interval
-        sleep(interval) if interval
-      end
-
       class << self
         #
         # Initialize an analyzer by query params
@@ -101,9 +114,7 @@ module Mihari
           query = copied[:query]
 
           # delete analyzer and query
-          %i[analyzer query].each do |key|
-            copied.delete key
-          end
+          %i[analyzer query].each { |key| copied.delete key }
 
           copied[:options] = copied[:options] || nil
 

data/lib/mihari/analyzers/binaryedge.rb

@@ -18,16 +18,7 @@ module Mihari
       end
 
       def artifacts
-        results = search
-        return [] unless results || results.empty?
-
-        results.map do |result|
-          events = result["events"] || []
-          events.filter_map do |event|
-            data = event.dig("target", "ip")
-            data.nil? ? nil : Artifact.new(data: data, source: source, metadata: event)
-          end
-        end.flatten
+        client.search_with_pagination(query, pagination_limit: pagination_limit).map(&:artifacts).flatten
       end
 
       def configuration_keys
@@ -36,49 +27,12 @@ module Mihari
 
       private
 
-      PAGE_SIZE = 20
-
-      #
-      # Search with pagination
-      #
-      # @param [Integer] page
-      #
-      # @return [Hash]
-      #
-      def search_with_page(page: 1)
-        client.search(query, page: page)
-      rescue StatusCodeError => e
-        raise RetryableError, e if e.message.include?("Request time limit exceeded")
-
-        raise e
-      end
-
-      #
-      # Search
-      #
-      # @return [Array<Hash>]
-      #
-      def search
-        responses = []
-        (1..pagination_limit).each do |page|
-          res = search_with_page(page: page)
-          total = res["total"].to_i
-
-          responses << res
-          break if total <= page * PAGE_SIZE
-
-          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
-          sleep_interval
-        end
-        responses
-      end
-
       #
       #
       # @return [Mihari::Clients::BinaryEdge]
       #
       def client
-        @client ||= Clients::BinaryEdge.new(api_key: api_key)
+        @client ||= Clients::BinaryEdge.new(api_key: api_key, interval: interval)
       end
     end
   end

data/lib/mihari/analyzers/censys.rb

@@ -27,25 +27,9 @@ module Mihari
       # @return [Array<Mihari::Artifact>]
       #
       def artifacts
-        artifacts = []
-
-        cursor = nil
-        pagination_limit.times do
-          response = client.search(query, cursor: cursor)
-          artifacts << response.result.to_artifacts
-          cursor = response.result.links.next
-          # NOTE: Censys's search API is unstable recently
-          # it may returns empty links or empty string cursors
-          # - Empty links: "links": {}
-          # - Empty cursors: "links": { "next": "", "prev": "" }
-          # So it needs to check both cases
-          break if cursor.nil? || cursor.empty?
-
-          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
-          sleep_interval
-        end
-
-        artifacts.flatten.uniq(&:data)
+        client.search_with_pagination(query, pagination_limit: pagination_limit).map do |res|
+          res.result.artifacts
+        end.flatten.uniq(&:data)
       end
 
       #
@@ -68,7 +52,7 @@ module Mihari
       # @return [Mihari::Clients::Censys]
       #
       def client
-        @client ||= Clients::Censys.new(id: id, secret: secret)
+        @client ||= Clients::Censys.new(id: id, secret: secret, interval: interval)
       end
 
       #

data/lib/mihari/analyzers/circl.rb

@@ -32,11 +32,11 @@ module Mihari
       def artifacts
         case type
         when "domain"
-          passive_dns_search
+          client.passive_dns_search query
         when "hash"
-          passive_ssl_search
+          client.passive_ssl_search query
         else
-          raise InvalidInputError, "#{@query}(type: #{@type || "unknown"}) is not supported."
+          raise ValueError, "#{@query}(type: #{@type || "unknown"}) is not supported."
         end
       end
 
@@ -54,30 +54,6 @@ module Mihari
         @client ||= Clients::CIRCL.new(username: username, password: password)
       end
 
-      #
-      # Passive DNS search
-      #
-      # @return [Array<String>]
-      #
-      def passive_dns_search
-        results = client.dns_query(query)
-        results.filter_map do |result|
-          type = result["rrtype"]
-          (type == "A") ? result["rdata"] : nil
-        end.uniq
-      end
-
-      #
-      # Passive SSL search
-      #
-      # @return [Array<String>]
-      #
-      def passive_ssl_search
-        result = client.ssl_cquery(query)
-        seen = result["seen"] || []
-        seen.uniq
-      end
-
       def username?
         !username.nil?
       end

data/lib/mihari/analyzers/crtsh.rb

@@ -18,13 +18,8 @@ module Mihari
       end
 
       def artifacts
-        results = search
-        results.map do |result|
-          values = result["name_value"].to_s.lines.map(&:chomp)
-          values.map do |value|
-            Artifact.new(data: value, source: source, metadata: result)
-          end
-        end.flatten
+        exclude = exclude_expired ? "expired" : nil
+        client.search(query, exclude: exclude)
       end
 
       private
@@ -35,16 +30,6 @@ module Mihari
       def client
         @client ||= Mihari::Clients::Crtsh.new
       end
-
-      #
-      # Search
-      #
-      # @return [Array<Hash>]
-      #
-      def search
-        exclude = exclude_expired ? "expired" : nil
-        client.search(query, exclude: exclude)
-      end
     end
   end
 end

data/lib/mihari/analyzers/dnstwister.rb

@@ -19,11 +19,9 @@ module Mihari
       end
 
       def artifacts
-        raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
+        raise ValueError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
 
-        res = client.fuzz(query)
-        fuzzy_domains = res["fuzzy_domains"] || []
-        domains = fuzzy_domains.map { |domain| domain["domain"] }
+        domains = client.fuzz(query)
         Parallel.map(domains) do |domain|
           resolvable?(domain) ? domain : nil
         end.compact

data/lib/mihari/analyzers/greynoise.rb

@@ -3,8 +3,6 @@
 module Mihari
   module Analyzers
     class GreyNoise < Base
-      PAGE_SIZE = 10_000
-
       # @return [String, nil]
       attr_reader :api_key
 
@@ -20,7 +18,10 @@ module Mihari
       end
 
       def artifacts
-        client.gnql_search(query, size: PAGE_SIZE).to_artifacts
+        client.gnql_search_with_pagination(
+          query,
+          pagination_limit: pagination_limit
+        ).map(&:artifacts).flatten
       end
 
       def configuration_keys
@@ -30,7 +31,7 @@ module Mihari
       private
 
       def client
-        @client ||= Clients::GreyNoise.new(api_key: api_key)
+        @client ||= Clients::GreyNoise.new(api_key: api_key, interval: interval)
       end
     end
   end

data/lib/mihari/analyzers/hunterhow.rb

@@ -30,25 +30,13 @@ module Mihari
       # @return [Array<Mihari::Artifact>]
       #
       def artifacts
-        artifacts = []
-
-        (1..pagination_limit).each do |page|
-          res = client.search(
-            query,
-            page: page,
-            page_size: PAGE_SIZE,
-            start_time: start_time.strftime("%Y-%m-%d"),
-            end_time: end_time.strftime("%Y-%m-%d")
-          )
-
-          artifacts << res.data.artifacts
-
-          break if res.data.list.length < PAGE_SIZE
-
-          sleep_interval
-        end
-
-        artifacts.flatten
+        client.search_with_pagination(
+          query,
+          start_time: start_time.strftime("%Y-%m-%d"),
+          end_time: end_time.strftime("%Y-%m-%d")
+        ).map do |res|
+          res.data.artifacts
+        end.flatten
       end
 
       def configuration_keys
@@ -57,11 +45,8 @@ module Mihari
 
       private
 
-      # @return [Integer]
-      PAGE_SIZE = 100
-
       def client
-        @client ||= Clients::HunterHow.new(api_key: api_key)
+        @client ||= Clients::HunterHow.new(api_key: api_key, interval: interval)
       end
     end
   end

data/lib/mihari/analyzers/onyphe.rb

@@ -20,10 +20,10 @@ module Mihari
       end
 
       def artifacts
-        responses = search
-        return [] unless responses
-
-        responses.map(&:to_artifacts).flatten
+        client.datascan_with_pagination(
+          query,
+          pagination_limit: pagination_limit
+        ).map(&:artifacts).flatten
       end
 
       def configuration_keys
@@ -32,42 +32,8 @@ module Mihari
 
       private
 
-      PAGE_SIZE = 10
-
       def client
-        @client ||= Clients::Onyphe.new(api_key: api_key)
-      end
-
-      #
-      # Search with pagination
-      #
-      # @param [String] query
-      # @param [Integer] page
-      #
-      # @return [Structs::Onyphe::Response]
-      #
-      def search_with_page(query, page: 1)
-        client.datascan(query, page: page)
-      end
-
-      #
-      # Search
-      #
-      # @return [Array<Structs::Onyphe::Response>]
-      #
-      def search
-        responses = []
-        (1..pagination_limit).each do |page|
-          res = search_with_page(query, page: page)
-          responses << res
-
-          total = res.total
-          break if total <= page * PAGE_SIZE
-
-          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
-          sleep_interval
-        end
-        responses
+        @client ||= Clients::Onyphe.new(api_key: api_key, interval: interval)
       end
     end
   end

data/lib/mihari/analyzers/otx.rb

@@ -27,11 +27,11 @@ module Mihari
       def artifacts
         case type
         when "domain"
-          domain_search
+          client.domain_search(query)
         when "ip"
-          ip_search
+          client.ip_search(query)
         else
-          raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
+          raise ValueError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
         end
       end
 
@@ -53,42 +53,6 @@ module Mihari
       def valid_type?
         %w[ip domain].include? type
       end
-
-      #
-      # Domain search
-      #
-      # @return [Array<String>]
-      #
-      def domain_search
-        res = client.query_by_domain(query)
-        return [] if res.nil?
-
-        records = res["passive_dns"] || []
-        records.filter_map do |record|
-          record_type = record["record_type"]
-          address = record["address"]
-
-          address if record_type == "A"
-        end.uniq
-      end
-
-      #
-      # IP search
-      #
-      # @return [Array<String>]
-      #
-      def ip_search
-        res = client.query_by_ip(query)
-        return [] if res.nil?
-
-        records = res["passive_dns"] || []
-        records.filter_map do |record|
-          record_type = record["record_type"]
-          hostname = record["hostname"]
-
-          hostname if record_type == "A"
-        end.uniq
-      end
     end
   end
 end

data/lib/mihari/analyzers/passivetotal.rb

@@ -32,13 +32,13 @@ module Mihari
       def artifacts
         case type
         when "domain", "ip"
-          passive_dns_search
+          client.passive_dns_search query
         when "mail"
-          reverse_whois_search
+          client.reverse_whois_search query
         when "hash"
-          ssl_search
+          client.ssl_search query
         else
-          raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
+          raise ValueError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
         end
       end
 
@@ -65,44 +65,6 @@ module Mihari
         %w[ip domain mail hash].include? type
       end
 
-      #
-      # Passive DNS search
-      #
-      # @return [Array<String>]
-      #
-      def passive_dns_search
-        res = client.passive_dns_search(query)
-        res["results"] || []
-      end
-
-      #
-      # Reverse whois search
-      #
-      # @return [Array<Mihari::Artifact>]
-      #
-      def reverse_whois_search
-        res = client.reverse_whois_search(query: query, field: "email")
-        results = res["results"] || []
-        results.map do |result|
-          data = result["domain"]
-          Artifact.new(data: data, source: source, metadata: result)
-        end.flatten
-      end
-
-      #
-      # Passive SSL search
-      #
-      # @return [Array<Mihari::Artifact>]
-      #
-      def ssl_search
-        res = client.ssl_search(query)
-        results = res["results"] || []
-        results.map do |result|
-          data = result["ipAddresses"]
-          data.map { |d| Artifact.new(data: d, source: source, metadata: result) }
-        end.flatten
-      end
-
       def username?
         !username.nil?
       end

data/lib/mihari/analyzers/pulsedive.rb

@@ -25,7 +25,7 @@ module Mihari
       end
 
       def artifacts
-        raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
+        raise ValueError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
 
         indicator = client.get_indicator(query)
         iid = indicator["iid"]