mihari 5.3.2 → 5.4.0

data/frontend/package.json

@@ -19,7 +19,7 @@
     "@fortawesome/vue-fontawesome": "^3.0.3",
     "@vueuse/core": "^10.3.0",
     "@vueuse/router": "^10.3.0",
-    "ace-builds": "^1.23.4",
+    "ace-builds": "^1.24.0",
     "axios": "^1.4.0",
     "bulma": "^0.9.4",
     "bulma-helpers": "^0.4.3",
@@ -37,14 +37,14 @@
     "vue3-ace-editor": "^2.2.3"
   },
   "devDependencies": {
-    "@redocly/cli": "1.0.0",
-    "@rushstack/eslint-patch": "^1.3.2",
+    "@redocly/cli": "1.0.2",
+    "@rushstack/eslint-patch": "^1.3.3",
     "@tsconfig/node20": "^20.1.1",
     "@types/jsdom": "^21.1.1",
-    "@types/node": "^20.4.8",
+    "@types/node": "^20.4.9",
     "@types/url-parse": "^1.4.8",
-    "@typescript-eslint/eslint-plugin": "^6.2.1",
-    "@typescript-eslint/parser": "^6.2.1",
+    "@typescript-eslint/eslint-plugin": "^6.3.0",
+    "@typescript-eslint/parser": "^6.3.0",
     "@vitejs/plugin-vue": "^4.2.3",
     "@vue/eslint-config-prettier": "^8.0.0",
     "@vue/eslint-config-typescript": "^11.0.3",
@@ -54,7 +54,7 @@
     "eslint-config-prettier": "^9.0.0",
     "eslint-plugin-prettier": "^5.0.0",
     "eslint-plugin-simple-import-sort": "^10.0.0",
-    "eslint-plugin-vue": "^9.16.1",
+    "eslint-plugin-vue": "^9.17.0",
     "husky": "^8.0.3",
     "jsdom": "^22.1.0",
     "npm-run-all": "^4.1.5",

data/lib/mihari/analyzers/base.rb

@@ -25,21 +25,28 @@ module Mihari
       # @return [Integer, nil]
       #
       def interval
-        @interval = options[:interval]
+        @interval ||= options[:interval]
       end
 
       #
       # @return [Integer]
       #
       def retry_interval
-        @retry_interval ||= options[:retry_interval] || DEFAULT_RETRY_INTERVAL
+        @retry_interval ||= options[:retry_interval] || Mihari.config.retry_interval
       end
 
       #
       # @return [Integer]
       #
       def retry_times
-        @retry_times ||= options[:retry_times] || DEFAULT_RETRY_TIMES
+        @retry_times ||= options[:retry_times] || Mihari.config.retry_times
+      end
+
+      #
+      # @return [Integer]
+      #
+      def pagination_limit
+        @pagination_limit ||= options[:pagination_limit] || Mihari.config.pagination_limit
       end
 
       # @return [Array<String>, Array<Mihari::Artifact>]

data/lib/mihari/analyzers/binaryedge.rb

@@ -56,7 +56,7 @@ module Mihari
       #
       def search
         responses = []
-        (1..500).each do |page|
+        (1..pagination_limit).each do |page|
           res = search_with_page(page: page)
           total = res["total"].to_i
 

data/lib/mihari/analyzers/censys.rb

@@ -30,7 +30,7 @@ module Mihari
         artifacts = []
 
         cursor = nil
-        loop do
+        pagination_limit.times do
           response = client.search(query, cursor: cursor)
           artifacts << response.result.to_artifacts
           cursor = response.result.links.next

data/lib/mihari/analyzers/hunterhow.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Analyzers
+    class HunterHow < Base
+      # @return [Integer]
+      PAGE_SIZE = 100
+
+      # @return [String, nil]
+      attr_reader :api_key
+
+      # @return [Date]
+      attr_reader :start_time
+
+      # @return [Date]
+      attr_reader :end_time
+
+      #
+      # @param [String] query
+      # @param [Hash, nil] options
+      # @param [String, nil] api_key
+      #
+      def initialize(query, start_time:, end_time:, options: nil, api_key: nil)
+        super(query, options: options)
+
+        @api_key = api_key || Mihari.config.hunterhow_api_key
+
+        @start_time = start_time
+        @end_time = end_time
+      end
+
+      #
+      # @return [Array<Mihari::Artifact>]
+      #
+      def artifacts
+        artifacts = []
+
+        (1..pagination_limit).each do |page|
+          res = client.search(
+            query,
+            page: page,
+            page_size: PAGE_SIZE,
+            start_time: start_time.strftime("%Y-%m-%d"),
+            end_time: end_time.strftime("%Y-%m-%d")
+          )
+
+          artifacts << res.data.artifacts
+
+          break if res.data.list.length < PAGE_SIZE
+
+          sleep_interval
+        end
+
+        artifacts.flatten
+      end
+
+      private
+
+      def configuration_keys
+        %w[hunterhow_api_key]
+      end
+
+      def client
+        @client ||= Clients::HunterHow.new(api_key: api_key)
+      end
+    end
+  end
+end

data/lib/mihari/analyzers/onyphe.rb

@@ -57,7 +57,7 @@ module Mihari
       #
       def search
         responses = []
-        (1..Float::INFINITY).each do |page|
+        (1..pagination_limit).each do |page|
           res = search_with_page(query, page: page)
           responses << res
 

data/lib/mihari/analyzers/rule.rb

@@ -10,6 +10,7 @@ module Mihari
       "dnstwister" => DNSTwister,
       "feed" => Feed,
       "greynoise" => GreyNoise,
+      "hunterhow" => HunterHow,
       "onyphe" => Onyphe,
       "otx" => OTX,
       "passivetotal" => PassiveTotal,

data/lib/mihari/analyzers/shodan.rb

@@ -54,7 +54,7 @@ module Mihari
       #
       def search
         responses = []
-        (1..Float::INFINITY).each do |page|
+        (1..pagination_limit).each do |page|
           res = search_with_page(page: page)
           responses << res
           break if res.total <= page * PAGE_SIZE

data/lib/mihari/analyzers/urlscan.rb

@@ -68,7 +68,7 @@ module Mihari
         responses = []
 
         search_after = nil
-        loop do
+        pagination_limit.times do
           res = search_with_search_after(search_after: search_after)
           responses << res
 

data/lib/mihari/analyzers/virustotal_intelligence.rb

@@ -45,7 +45,7 @@ module Mihari
         cursor = nil
         responses = []
 
-        loop do
+        pagination_limit.times do
           response = Structs::VirusTotalIntelligence::Response.from_dynamic!(client.intel_search(query,
             cursor: cursor))
           responses << response

data/lib/mihari/analyzers/zoomeye.rb

@@ -95,7 +95,7 @@ module Mihari
       #
       def host_search
         responses = []
-        (1..Float::INFINITY).each do |page|
+        (1..pagination_limit).each do |page|
           res = _host_search(query, page: page)
           break unless res
 
@@ -128,7 +128,7 @@ module Mihari
       #
       def web_search
         responses = []
-        (1..Float::INFINITY).each do |page|
+        (1..pagination_limit).each do |page|
           res = _web_search(query, page: page)
           break unless res
 

data/lib/mihari/clients/hunterhow.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require "base64"
+
+module Mihari
+  module Clients
+    class HunterHow < Base
+      # @return [String]
+      attr_reader :api_key
+
+      #
+      # @param [String] base_url
+      # @param [String, nil] api_key
+      # @param [Hash] headers
+      #
+      def initialize(base_url = "https://api.hunter.how/", api_key:, headers: {})
+        raise(ArgumentError, "'api_key' argument is required") unless api_key
+
+        super(base_url, headers: headers)
+
+        @api_key = api_key
+      end
+
+      #
+      # @param [String] query String used to query our data
+      # @param [Integer] page Default 1, Maximum: 500
+      # @param [Integer] page_size Default 100, Maximum: 100
+      # @param [String] start_time
+      # @param [String] end_time
+      #
+      # @return [Structs::HunterHow::Response]
+      #
+      def search(query, start_time:, end_time:, page: 1, page_size: 10)
+        params = {
+          query: Base64.urlsafe_encode64(query),
+          page: page,
+          page_size: page_size,
+          start_time: start_time,
+          end_time: end_time,
+          "api-key": api_key
+        }.compact
+        res = get("/search", params: params)
+        Structs::HunterHow::Response.from_dynamic! JSON.parse(res.body.to_s)
+      end
+    end
+  end
+end

data/lib/mihari/config.rb

@@ -1,85 +1,97 @@
 module Mihari
   class Config
     # @return [String, nil]
-    attr_accessor :binaryedge_api_key
+    attr_reader :binaryedge_api_key
 
     # @return [String, nil]
-    attr_accessor :censys_id
+    attr_reader :censys_id
 
     # @return [String, nil]
-    attr_accessor :censys_secret
+    attr_reader :censys_secret
 
     # @return [String, nil]
-    attr_accessor :circl_passive_password
+    attr_reader :circl_passive_password
 
     # @return [String, nil]
-    attr_accessor :circl_passive_username
+    attr_reader :circl_passive_username
 
     # @return [URI]
-    attr_accessor :database_url
+    attr_reader :database_url
 
     # @return [String, nil]
-    attr_accessor :greynoise_api_key
+    attr_reader :greynoise_api_key
 
     # @return [String, nil]
-    attr_accessor :ipinfo_api_key
+    attr_reader :hunterhow_api_key
 
     # @return [String, nil]
-    attr_accessor :misp_url
+    attr_reader :ipinfo_api_key
 
     # @return [String, nil]
-    attr_accessor :misp_api_key
+    attr_reader :misp_url
 
     # @return [String, nil]
-    attr_accessor :onyphe_api_key
+    attr_reader :misp_api_key
 
     # @return [String, nil]
-    attr_accessor :otx_api_key
+    attr_reader :onyphe_api_key
 
     # @return [String, nil]
-    attr_accessor :passivetotal_api_key
+    attr_reader :otx_api_key
 
     # @return [String, nil]
-    attr_accessor :passivetotal_username
+    attr_reader :passivetotal_api_key
 
     # @return [String, nil]
-    attr_accessor :pulsedive_api_key
+    attr_reader :passivetotal_username
 
     # @return [String, nil]
-    attr_accessor :securitytrails_api_key
+    attr_reader :pulsedive_api_key
 
     # @return [String, nil]
-    attr_accessor :shodan_api_key
+    attr_reader :securitytrails_api_key
 
     # @return [String, nil]
-    attr_accessor :slack_channel
+    attr_reader :shodan_api_key
 
     # @return [String, nil]
-    attr_accessor :slack_webhook_url
+    attr_reader :slack_channel
 
     # @return [String, nil]
-    attr_accessor :thehive_url
+    attr_reader :slack_webhook_url
 
     # @return [String, nil]
-    attr_accessor :thehive_api_key
+    attr_reader :thehive_url
 
     # @return [String, nil]
-    attr_accessor :thehive_api_version
+    attr_reader :thehive_api_key
 
     # @return [String, nil]
-    attr_accessor :urlscan_api_key
+    attr_reader :thehive_api_version
 
     # @return [String, nil]
-    attr_accessor :virustotal_api_key
+    attr_reader :urlscan_api_key
 
     # @return [String, nil]
-    attr_accessor :zoomeye_api_key
+    attr_reader :virustotal_api_key
 
     # @return [String, nil]
-    attr_accessor :sentry_dsn
+    attr_reader :zoomeye_api_key
 
     # @return [String, nil]
-    attr_accessor :hide_config_values
+    attr_reader :sentry_dsn
+
+    # @return [Boolean]
+    attr_reader :hide_config_values
+
+    # @return [Integer]
+    attr_reader :retry_interval
+
+    # @return [Integer]
+    attr_reader :retry_times
+
+    # @return [Integer]
+    attr_reader :pagination_limit
 
     def initialize
       @binaryedge_api_key = ENV.fetch("BINARYEDGE_API_KEY", nil)
@@ -96,6 +108,8 @@ module Mihari
 
       @ipinfo_api_key = ENV.fetch("IPINFO_API_KEY", nil)
 
+      @hunterhow_api_key = ENV.fetch("HUNTERHOW_API_KEY", nil)
+
       @misp_url = ENV.fetch("MISP_URL", nil)
       @misp_api_key = ENV.fetch("MISP_API_KEY", nil)
 
@@ -128,6 +142,11 @@ module Mihari
       @sentry_dsn = ENV.fetch("SENTRY_DSN", nil)
 
       @hide_config_values = ENV.fetch("HIDE_CONFIG_VALUES", false)
+
+      @retry_times = ENV.fetch("RETRY_TIMES", 3).to_i
+      @retry_interval = ENV.fetch("RETRY_INTERVAL", 5).to_i
+
+      @pagination_limit = ENV.fetch("PAGINATION_LIMIT", 1000).to_i
     end
   end
 end

data/lib/mihari/constants.rb

@@ -9,7 +9,4 @@ module Mihari
 
   # @return [Array<Hash>]
   DEFAULT_ENRICHERS = %w[whois ipinfo shodan google_public_dns].map { |name| { enricher: name } }.freeze
-
-  DEFAULT_RETRY_TIMES = 3
-  DEFAULT_RETRY_INTERVAL = 5
 end

data/lib/mihari/schemas/analyzer.rb

@@ -4,8 +4,9 @@ module Mihari
   module Schemas
     AnalyzerOptions = Dry::Schema.Params do
       optional(:interval).value(:integer)
-      optional(:retry_times).value(:integer).default(DEFAULT_RETRY_TIMES)
-      optional(:retry_interval).value(:integer).default(DEFAULT_RETRY_INTERVAL)
+      optional(:pagination_limit).value(:integer).default(Mihari.config.pagination_limit)
+      optional(:retry_times).value(:integer).default(Mihari.config.retry_times)
+      optional(:retry_interval).value(:integer).default(Mihari.config.retry_interval)
     end
 
     AnalyzerWithoutAPIKey = Dry::Schema.Params do
@@ -75,6 +76,15 @@ module Mihari
       optional(:options).hash(AnalyzerOptions)
     end
 
+    HunterHow = Dry::Schema.Params do
+      required(:analyzer).value(Types::String.enum("hunterhow"))
+      required(:query).value(:string)
+      required(:start_time).value(:date)
+      required(:end_time).value(:date)
+      optional(:api_key).value(:string)
+      optional(:options).hash(AnalyzerOptions)
+    end
+
     Feed = Dry::Schema.Params do
       required(:analyzer).value(Types::String.enum("feed"))
       required(:query).value(:string)

data/lib/mihari/schemas/rule.rb

@@ -22,7 +22,7 @@ module Mihari
       optional(:updated_on).value(:date)
 
       required(:queries).value(:array).each do
-        AnalyzerWithoutAPIKey | AnalyzerWithAPIKey | Censys | CIRCL | PassiveTotal | ZoomEye | Crtsh | Feed
+        AnalyzerWithoutAPIKey | AnalyzerWithAPIKey | Censys | CIRCL | PassiveTotal | ZoomEye | Crtsh | Feed | HunterHow
       end
 
       optional(:emitters).value(:array).each { Database | MISP | TheHive | Slack | Webhook }.default(DEFAULT_EMITTERS)

data/lib/mihari/structs/hunterhow.rb

@@ -0,0 +1,104 @@
+module Mihari
+  module Structs
+    module HunterHow
+      class ListItem < Dry::Struct
+        attribute :domain, Types::String
+        attribute :ip, Types::String
+        attribute :port, Types::Integer
+
+        #
+        # @return [String]
+        #
+        def ip
+          attributes[:ip]
+        end
+
+        #
+        # @return [Mihari::Artifact]
+        #
+        def artifact
+          Artifact.new(data: ip)
+        end
+
+        class << self
+          #
+          # @param [Hash] d
+          #
+          # @return [ListItem]
+          #
+          def from_dynamic!(d)
+            d = Types::Hash[d]
+            new(
+              domain: d.fetch("domain"),
+              ip: d.fetch("ip"),
+              port: d.fetch("port")
+            )
+          end
+        end
+      end
+
+      class DataClass < Dry::Struct
+        attribute :list, Types.Array(ListItem)
+        attribute :total, Types::Integer
+
+        #
+        # @return [Array<ListItem>]
+        #
+        def list
+          attributes[:list]
+        end
+
+        #
+        # @return [Array<Mihari::Artifact>]
+        #
+        def artifacts
+          list.map(&:artifact)
+        end
+
+        class << self
+          #
+          # @param [Hash] d
+          #
+          # @return [DataClass]
+          #
+          def from_dynamic!(d)
+            d = Types::Hash[d]
+            new(
+              list: d.fetch("list").map { |x| ListItem.from_dynamic!(x) },
+              total: d.fetch("total")
+            )
+          end
+        end
+      end
+
+      class Response < Dry::Struct
+        attribute :code, Types::Integer
+        attribute :data, DataClass
+        attribute :message, Types::String
+
+        #
+        # @return [DataClass]
+        #
+        def data
+          attributes[:data]
+        end
+
+        class << self
+          #
+          # @param [Hash] d
+          #
+          # @return [Response]
+          #
+          def from_dynamic!(d)
+            d = Types::Hash[d]
+            new(
+              code: d.fetch("code"),
+              data: DataClass.from_dynamic!(d.fetch("data")),
+              message: d.fetch("message")
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "5.3.2"
+  VERSION = "5.4.0"
 end