mihari 5.3.1 → 5.4.0

data/lib/mihari/analyzers/zoomeye.rb

@@ -95,7 +95,7 @@ module Mihari
       #
       def host_search
         responses = []
-        (1..Float::INFINITY).each do |page|
+        (1..pagination_limit).each do |page|
           res = _host_search(query, page: page)
           break unless res
 
@@ -104,7 +104,7 @@ module Mihari
           break if total <= page * PAGE_SIZE
 
           # sleep #{interval} seconds to avoid the rate limitation (if it is set)
-          sleep(interval) if interval
+          sleep_interval
         end
         convert_responses responses.compact
       end
@@ -128,7 +128,7 @@ module Mihari
       #
       def web_search
         responses = []
-        (1..Float::INFINITY).each do |page|
+        (1..pagination_limit).each do |page|
           res = _web_search(query, page: page)
           break unless res
 
@@ -137,7 +137,7 @@ module Mihari
           break if total <= page * PAGE_SIZE
 
           # sleep #{interval} seconds to avoid the rate limitation (if it is set)
-          sleep(interval) if interval
+          sleep_interval
         end
         convert_responses responses.compact
       end

data/lib/mihari/clients/hunterhow.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require "base64"
+
+module Mihari
+  module Clients
+    class HunterHow < Base
+      # @return [String]
+      attr_reader :api_key
+
+      #
+      # @param [String] base_url
+      # @param [String, nil] api_key
+      # @param [Hash] headers
+      #
+      def initialize(base_url = "https://api.hunter.how/", api_key:, headers: {})
+        raise(ArgumentError, "'api_key' argument is required") unless api_key
+
+        super(base_url, headers: headers)
+
+        @api_key = api_key
+      end
+
+      #
+      # @param [String] query String used to query our data
+      # @param [Integer] page Default 1, Maximum: 500
+      # @param [Integer] page_size Default 100, Maximum: 100
+      # @param [String] start_time
+      # @param [String] end_time
+      #
+      # @return [Structs::HunterHow::Response]
+      #
+      def search(query, start_time:, end_time:, page: 1, page_size: 10)
+        params = {
+          query: Base64.urlsafe_encode64(query),
+          page: page,
+          page_size: page_size,
+          start_time: start_time,
+          end_time: end_time,
+          "api-key": api_key
+        }.compact
+        res = get("/search", params: params)
+        Structs::HunterHow::Response.from_dynamic! JSON.parse(res.body.to_s)
+      end
+    end
+  end
+end

data/lib/mihari/commands/rule.rb

@@ -15,7 +15,7 @@ module Mihari
             # @param [String] path
             #
             def validate(path)
-              rule = Structs::Rule.from_path_or_id(path)
+              rule = Services::Rule.from_path_or_id(path)
 
               begin
                 rule.validate!
@@ -44,10 +44,10 @@ module Mihari
 
             no_commands do
               #
-              # @return [Mihari::Structs::Rule]
+              # @return [Mihari::Services::Rule]
               #
               def rule_template
-                Structs::Rule.from_path File.expand_path("../templates/rule.yml.erb", __dir__)
+                Services::Rule.from_path File.expand_path("../templates/rule.yml.erb", __dir__)
               end
 
               #

data/lib/mihari/commands/search.rb

@@ -33,12 +33,12 @@ module Mihari
           end
 
           def update_or_create
-            rule.model.save
+            rule.to_model.save
           end
 
           def run
             begin
-              analyzer = rule.analyzer
+              analyzer = rule.to_analyzer
             rescue ConfigurationError => e
               # if there is a configuration error, output that error without the stack trace
               Mihari.logger.error e.to_s
@@ -69,7 +69,7 @@ module Mihari
             #
             def search(path_or_id)
               Mihari::Database.with_db_connection do
-                rule = Structs::Rule.from_path_or_id path_or_id
+                rule = Services::Rule.from_path_or_id path_or_id
 
                 begin
                   rule.validate!

data/lib/mihari/config.rb

@@ -1,85 +1,97 @@
 module Mihari
   class Config
     # @return [String, nil]
-    attr_accessor :binaryedge_api_key
+    attr_reader :binaryedge_api_key
 
     # @return [String, nil]
-    attr_accessor :censys_id
+    attr_reader :censys_id
 
     # @return [String, nil]
-    attr_accessor :censys_secret
+    attr_reader :censys_secret
 
     # @return [String, nil]
-    attr_accessor :circl_passive_password
+    attr_reader :circl_passive_password
 
     # @return [String, nil]
-    attr_accessor :circl_passive_username
+    attr_reader :circl_passive_username
 
     # @return [URI]
-    attr_accessor :database_url
+    attr_reader :database_url
 
     # @return [String, nil]
-    attr_accessor :greynoise_api_key
+    attr_reader :greynoise_api_key
 
     # @return [String, nil]
-    attr_accessor :ipinfo_api_key
+    attr_reader :hunterhow_api_key
 
     # @return [String, nil]
-    attr_accessor :misp_url
+    attr_reader :ipinfo_api_key
 
     # @return [String, nil]
-    attr_accessor :misp_api_key
+    attr_reader :misp_url
 
     # @return [String, nil]
-    attr_accessor :onyphe_api_key
+    attr_reader :misp_api_key
 
     # @return [String, nil]
-    attr_accessor :otx_api_key
+    attr_reader :onyphe_api_key
 
     # @return [String, nil]
-    attr_accessor :passivetotal_api_key
+    attr_reader :otx_api_key
 
     # @return [String, nil]
-    attr_accessor :passivetotal_username
+    attr_reader :passivetotal_api_key
 
     # @return [String, nil]
-    attr_accessor :pulsedive_api_key
+    attr_reader :passivetotal_username
 
     # @return [String, nil]
-    attr_accessor :securitytrails_api_key
+    attr_reader :pulsedive_api_key
 
     # @return [String, nil]
-    attr_accessor :shodan_api_key
+    attr_reader :securitytrails_api_key
 
     # @return [String, nil]
-    attr_accessor :slack_channel
+    attr_reader :shodan_api_key
 
     # @return [String, nil]
-    attr_accessor :slack_webhook_url
+    attr_reader :slack_channel
 
     # @return [String, nil]
-    attr_accessor :thehive_url
+    attr_reader :slack_webhook_url
 
     # @return [String, nil]
-    attr_accessor :thehive_api_key
+    attr_reader :thehive_url
 
     # @return [String, nil]
-    attr_accessor :thehive_api_version
+    attr_reader :thehive_api_key
 
     # @return [String, nil]
-    attr_accessor :urlscan_api_key
+    attr_reader :thehive_api_version
 
     # @return [String, nil]
-    attr_accessor :virustotal_api_key
+    attr_reader :urlscan_api_key
 
     # @return [String, nil]
-    attr_accessor :zoomeye_api_key
+    attr_reader :virustotal_api_key
 
     # @return [String, nil]
-    attr_accessor :sentry_dsn
+    attr_reader :zoomeye_api_key
 
     # @return [String, nil]
-    attr_accessor :hide_config_values
+    attr_reader :sentry_dsn
+
+    # @return [Boolean]
+    attr_reader :hide_config_values
+
+    # @return [Integer]
+    attr_reader :retry_interval
+
+    # @return [Integer]
+    attr_reader :retry_times
+
+    # @return [Integer]
+    attr_reader :pagination_limit
 
     def initialize
       @binaryedge_api_key = ENV.fetch("BINARYEDGE_API_KEY", nil)
@@ -96,6 +108,8 @@ module Mihari
 
       @ipinfo_api_key = ENV.fetch("IPINFO_API_KEY", nil)
 
+      @hunterhow_api_key = ENV.fetch("HUNTERHOW_API_KEY", nil)
+
       @misp_url = ENV.fetch("MISP_URL", nil)
       @misp_api_key = ENV.fetch("MISP_API_KEY", nil)
 
@@ -128,6 +142,11 @@ module Mihari
       @sentry_dsn = ENV.fetch("SENTRY_DSN", nil)
 
       @hide_config_values = ENV.fetch("HIDE_CONFIG_VALUES", false)
+
+      @retry_times = ENV.fetch("RETRY_TIMES", 3).to_i
+      @retry_interval = ENV.fetch("RETRY_INTERVAL", 5).to_i
+
+      @pagination_limit = ENV.fetch("PAGINATION_LIMIT", 1000).to_i
     end
   end
 end

data/lib/mihari/constants.rb

@@ -1,12 +1,12 @@
 # frozen_string_literal: true
 
 module Mihari
+  # @return [Array<String>]
   DEFAULT_DATA_TYPES = %w[hash ip domain url mail].freeze
 
+  # @return [Array<Hash>]
   DEFAULT_EMITTERS = %w[database misp slack the_hive].map { |name| { emitter: name } }.freeze
 
+  # @return [Array<Hash>]
   DEFAULT_ENRICHERS = %w[whois ipinfo shodan google_public_dns].map { |name| { enricher: name } }.freeze
-
-  DEFAULT_RETRY_TIMES = 3
-  DEFAULT_RETRY_INTERVAL = 5
 end

data/lib/mihari/emitters/base.rb

@@ -9,12 +9,12 @@ module Mihari
       # @return [Array<Mihari::Artifact>]
       attr_reader :artifacts
 
-      # @return [Mihari::Structs::Rule]
+      # @return [Mihari::Services::Rule]
       attr_reader :rule
 
       #
       # @param [Array<Mihari::Artifact>] artifacts
-      # @param [Mihari::Structs::Rule] rule
+      # @param [Mihari::Services::Rule] rule
       # @param [Hash] **_options
       #
       def initialize(artifacts:, rule:, **_options)

data/lib/mihari/emitters/misp.rb

@@ -12,12 +12,12 @@ module Mihari
       # @return [Array<Mihari::Artifact>]
       attr_reader :artifacts
 
-      # @return [Mihari::Structs::Rule]
+      # @return [Mihari::Services::Rule]
       attr_reader :rule
 
       #
       # @param [Array<Mihari::Artifact>] artifacts
-      # @param [Mihari::Structs::Rule] rule
+      # @param [Mihari::Services::Rule] rule
       # @param [Hash] **options
       #
       def initialize(artifacts:, rule:, **options)
@@ -47,7 +47,7 @@ module Mihari
       # Create a MISP event
       #
       # @param [Arra<Mihari::Artifact>] artifacts
-      # @param [Mihari::Structs::Rule] rule
+      # @param [Mihari::Services::Rule] rule
       #
       # @return [::MISP::Event]
       #

data/lib/mihari/emitters/slack.rb

@@ -133,7 +133,7 @@ module Mihari
 
       #
       # @param [Array<Mihari::Artifact>] artifacts
-      # @param [Mihari::Structs::Rule] rule
+      # @param [Mihari::Services::Rule] rule
       # @param [Hash] **_options
       #
       def initialize(artifacts:, rule:, **options)

data/lib/mihari/emitters/the_hive.rb

@@ -14,7 +14,7 @@ module Mihari
 
       #
       # @param [Array<Mihari::Artifact>] artifacts
-      # @param [Mihari::Structs::Rule] rule
+      # @param [Mihari::Services::Rule] rule
       # @param [Hash] **options
       #
       def initialize(artifacts:, rule:, **options)

data/lib/mihari/emitters/webhook.rb

@@ -57,7 +57,7 @@ module Mihari
 
       #
       # @param [Array<Mihari::Artifact>] artifacts
-      # @param [Mihari::Structs::Rule] rule
+      # @param [Mihari::Services::Rule] rule
       # @param [Hash] **options
       #
       def initialize(artifacts:, rule:, **options)

data/lib/mihari/mixins/configurable.rb

@@ -49,6 +49,11 @@ module Mihari
 
       private
 
+      #
+      # Check whether API key is set or not
+      #
+      # @return [Boolean]
+      #
       def api_key?
         value = method(:api_key).call
         !value.nil?

data/lib/mihari/mixins/falsepositive.rb

@@ -22,7 +22,7 @@ module Mihari
       memoize :normalize_falsepositive
 
       #
-      # Check whetehr a value is valid format as a disallowed data value
+      # Check whether a value is valid format as a disallowed data value
       #
       # @param [String] value Data value
       #

data/lib/mihari/mixins/retriable.rb

@@ -21,8 +21,6 @@ module Mihari
       # @param [Integer] interval
       # @param [Array<StandardError>] on
       #
-      # @return [nil]
-      #
       def retry_on_error(times: 3, interval: 5, on: DEFAULT_ON)
         try = 0
         begin

data/lib/mihari/schemas/analyzer.rb

@@ -4,8 +4,9 @@ module Mihari
   module Schemas
     AnalyzerOptions = Dry::Schema.Params do
       optional(:interval).value(:integer)
-      optional(:retry_times).value(:integer).default(DEFAULT_RETRY_TIMES)
-      optional(:retry_interval).value(:integer).default(DEFAULT_RETRY_INTERVAL)
+      optional(:pagination_limit).value(:integer).default(Mihari.config.pagination_limit)
+      optional(:retry_times).value(:integer).default(Mihari.config.retry_times)
+      optional(:retry_interval).value(:integer).default(Mihari.config.retry_interval)
     end
 
     AnalyzerWithoutAPIKey = Dry::Schema.Params do
@@ -75,6 +76,15 @@ module Mihari
       optional(:options).hash(AnalyzerOptions)
     end
 
+    HunterHow = Dry::Schema.Params do
+      required(:analyzer).value(Types::String.enum("hunterhow"))
+      required(:query).value(:string)
+      required(:start_time).value(:date)
+      required(:end_time).value(:date)
+      optional(:api_key).value(:string)
+      optional(:options).hash(AnalyzerOptions)
+    end
+
     Feed = Dry::Schema.Params do
       required(:analyzer).value(Types::String.enum("feed"))
       required(:query).value(:string)

data/lib/mihari/schemas/rule.rb

@@ -22,7 +22,7 @@ module Mihari
       optional(:updated_on).value(:date)
 
       required(:queries).value(:array).each do
-        AnalyzerWithoutAPIKey | AnalyzerWithAPIKey | Censys | CIRCL | PassiveTotal | ZoomEye | Crtsh | Feed
+        AnalyzerWithoutAPIKey | AnalyzerWithAPIKey | Censys | CIRCL | PassiveTotal | ZoomEye | Crtsh | Feed | HunterHow
       end
 
       optional(:emitters).value(:array).each { Database | MISP | TheHive | Slack | Webhook }.default(DEFAULT_EMITTERS)

data/lib/mihari/{structs → services}/rule.rb

@@ -8,7 +8,7 @@ require "securerandom"
 require "yaml"
 
 module Mihari
-  module Structs
+  module Services
     class Rule
       include Mixins::FalsePositive
 
@@ -49,16 +49,16 @@ module Mihari
       end
 
       def validate!
-        raise RuleValidationError if errors?
-      rescue RuleValidationError => e
+        return unless errors?
+
         Mihari.logger.error "Failed to parse the input as a rule:"
         Mihari.logger.error JSON.pretty_generate(errors.to_h)
 
-        raise e
+        raise RuleValidationError, errors
       end
 
       def [](key)
-        data[key.to_sym]
+        data key.to_sym
       end
 
       #
@@ -141,7 +141,7 @@ module Mihari
       #
       # @return [Mihari::Rule]
       #
-      def model
+      def to_model
         rule = Mihari::Rule.find(id)
 
         rule.title = title
@@ -161,8 +161,8 @@ module Mihari
       #
       # @return [Mihari::Analyzers::Rule]
       #
-      def analyzer
-        Mihari::Analyzers::Rule.new(self)
+      def to_analyzer
+        Mihari::Analyzers::Rule.new self
       end
 
       class << self
@@ -171,10 +171,10 @@ module Mihari
         #
         # @param [String] yaml
         #
-        # @return [Mihari::Structs::Rule]
+        # @return [Mihari::Services::Rule]
         #
         def from_yaml(yaml)
-          Structs::Rule.new YAML.safe_load(ERB.new(yaml).result, permitted_classes: [Date, Symbol])
+          Services::Rule.new YAML.safe_load(ERB.new(yaml).result, permitted_classes: [Date, Symbol])
         rescue Psych::SyntaxError => e
           raise YAMLSyntaxError, e.message
         end
@@ -182,10 +182,10 @@ module Mihari
         #
         # @param [Mihari::Rule] model
         #
-        # @return [Mihari::Structs::Rule]
+        # @return [Mihari::Services::Rule]
         #
         def from_model(model)
-          Structs::Rule.new(model.data)
+          Services::Rule.new model.data
         end
 
         #
@@ -193,7 +193,7 @@ module Mihari
         #
         # @param [String] path
         #
-        # @return [Mihari::Structs::Rule, nil]
+        # @return [Mihari::Services::Rule, nil]
         #
         def from_path(path)
           return nil unless Pathname(path).exist?
@@ -206,18 +206,18 @@ module Mihari
         #
         # @param [String] id
         #
-        # @return [Mihari::Structs::Rule, nil]
+        # @return [Mihari::Services::Rule, nil]
         #
         def from_id(id)
           return nil unless Mihari::Rule.exists?(id)
 
-          Structs::Rule.from_model Mihari::Rule.find(id)
+          Services::Rule.from_model Mihari::Rule.find(id)
         end
 
         #
         # @param [String] path_or_id Path to YAML file or YAML string or ID of a rule in the database
         #
-        # @return [Mihari::Structs::Rule]
+        # @return [Mihari::Services::Rule]
         #
         def from_path_or_id(path_or_id)
           rule = from_path(path_or_id)

data/lib/mihari/structs/hunterhow.rb

@@ -0,0 +1,104 @@
+module Mihari
+  module Structs
+    module HunterHow
+      class ListItem < Dry::Struct
+        attribute :domain, Types::String
+        attribute :ip, Types::String
+        attribute :port, Types::Integer
+
+        #
+        # @return [String]
+        #
+        def ip
+          attributes[:ip]
+        end
+
+        #
+        # @return [Mihari::Artifact]
+        #
+        def artifact
+          Artifact.new(data: ip)
+        end
+
+        class << self
+          #
+          # @param [Hash] d
+          #
+          # @return [ListItem]
+          #
+          def from_dynamic!(d)
+            d = Types::Hash[d]
+            new(
+              domain: d.fetch("domain"),
+              ip: d.fetch("ip"),
+              port: d.fetch("port")
+            )
+          end
+        end
+      end
+
+      class DataClass < Dry::Struct
+        attribute :list, Types.Array(ListItem)
+        attribute :total, Types::Integer
+
+        #
+        # @return [Array<ListItem>]
+        #
+        def list
+          attributes[:list]
+        end
+
+        #
+        # @return [Array<Mihari::Artifact>]
+        #
+        def artifacts
+          list.map(&:artifact)
+        end
+
+        class << self
+          #
+          # @param [Hash] d
+          #
+          # @return [DataClass]
+          #
+          def from_dynamic!(d)
+            d = Types::Hash[d]
+            new(
+              list: d.fetch("list").map { |x| ListItem.from_dynamic!(x) },
+              total: d.fetch("total")
+            )
+          end
+        end
+      end
+
+      class Response < Dry::Struct
+        attribute :code, Types::Integer
+        attribute :data, DataClass
+        attribute :message, Types::String
+
+        #
+        # @return [DataClass]
+        #
+        def data
+          attributes[:data]
+        end
+
+        class << self
+          #
+          # @param [Hash] d
+          #
+          # @return [Response]
+          #
+          def from_dynamic!(d)
+            d = Types::Hash[d]
+            new(
+              code: d.fetch("code"),
+              data: DataClass.from_dynamic!(d.fetch("data")),
+              message: d.fetch("message")
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "5.3.1"
+  VERSION = "5.4.0"
 end