mihari 5.2.3 → 5.3.0

data/lib/mihari/analyzers/shodan.rb

@@ -3,23 +3,18 @@
 module Mihari
   module Analyzers
     class Shodan < Base
-      param :query
-
-      option :interval, default: proc { 0 }
-
       # @return [String, nil]
       attr_reader :api_key
 
-      # @return [Integer]
-      attr_reader :interval
-
-      # @return [String]
-      attr_reader :query
-
-      def initialize(*args, **kwargs)
-        super(*args, **kwargs)
+      #
+      # @param [String] query
+      # @param [Hash, nil] options
+      # @param [String, nil] api_key
+      #
+      def initialize(query, options: nil, api_key: nil)
+        super(query, options: options)
 
-        @api_key = kwargs[:api_key] || Mihari.config.shodan_api_key
+        @api_key = api_key || Mihari.config.shodan_api_key
       end
 
       def artifacts

data/lib/mihari/analyzers/urlscan.rb

@@ -3,35 +3,30 @@
 module Mihari
   module Analyzers
     class Urlscan < Base
-      param :query
-
-      option :allowed_data_types, default: proc { SUPPORTED_DATA_TYPES }
-
-      option :interval, default: proc { 0 }
-
       SUPPORTED_DATA_TYPES = %w[url domain ip].freeze
       SIZE = 1000
 
       # @return [String, nil]
       attr_reader :api_key
 
-      # @return [String]
-      attr_reader :query
-
-      # @return [Integer]
-      attr_reader :interval
-
-      # @return [String]
+      # @return [Array<String>]
       attr_reader :allowed_data_types
 
-      def initialize(*args, **kwargs)
-        super
+      #
+      # @param [String] query
+      # @param [Hash, nil] options
+      # @param [String, nil] api_key
+      # @param [Array<String>] allowed_data_types
+      #
+      def initialize(query, options: nil, api_key: nil, allowed_data_types: SUPPORTED_DATA_TYPES)
+        super(query, options: options)
 
-        unless valid_allowed_data_types?
-          raise InvalidInputError, "allowed_data_types should be any of url, domain and ip."
-        end
+        @api_key = api_key || Mihari.config.urlscan_api_key
+        @allowed_data_types = allowed_data_types
+
+        return if valid_allowed_data_types?
 
-        @api_key = kwargs[:api_key] || Mihari.config.urlscan_api_key
+        raise InvalidInputError, "allowed_data_types should be any of url, domain and ip."
       end
 
       def artifacts
@@ -82,7 +77,7 @@ module Mihari
           search_after = res.results.last.sort.join(",")
 
           # sleep #{interval} seconds to avoid the rate limitation (if it is set)
-          sleep interval
+          sleep(interval) if interval
         end
 
         responses

data/lib/mihari/analyzers/virustotal.rb

@@ -5,28 +5,34 @@ module Mihari
     class VirusTotal < Base
       include Mixins::Refang
 
-      param :query
-
       # @return [String]
       attr_reader :type
 
       # @return [String, nil]
       attr_reader :api_key
 
-      # @return [String]
-      attr_reader :query
-
-      def initialize(*args, **kwargs)
-        super(*args, **kwargs)
+      #
+      # @param [String] query
+      # @param [Hash, nil] options
+      # @param [String, nil] api_key
+      #
+      def initialize(query, options: nil, api_key: nil)
+        super(refang(query), options: options)
 
-        @query = refang(query)
         @type = TypeChecker.type(query)
 
-        @api_key = kwargs[:api_key] || Mihari.config.virustotal_api_key
+        @api_key = api_key || Mihari.config.virustotal_api_key
       end
 
       def artifacts
-        search || []
+        case type
+        when "domain"
+          domain_search
+        when "ip"
+          ip_search
+        else
+          raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
+        end
       end
 
       private
@@ -48,22 +54,6 @@ module Mihari
         %w[ip domain].include? type
       end
 
-      #
-      # Search
-      #
-      # @return [Array<Mihari::Artifact>]
-      #
-      def search
-        case type
-        when "domain"
-          domain_search
-        when "ip"
-          ip_search
-        else
-          raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
-        end
-      end
-
       #
       # Domain search
       #

data/lib/mihari/analyzers/virustotal_intelligence.rb

@@ -3,25 +3,18 @@
 module Mihari
   module Analyzers
     class VirusTotalIntelligence < Base
-      param :query
-
-      option :interval, default: proc { 0 }
-
       # @return [String, nil]
       attr_reader :api_key
 
-      # @return [String]
-      attr_reader :query
-
-      # @return [Integer]
-      attr_reader :interval
-
-      def initialize(*args, **kwargs)
-        super
-
-        @query = query
+      #
+      # @param [String] query
+      # @param [Hash, nll] options
+      # @param [String, nil] api_key
+      #
+      def initialize(query, options: nil, api_key: nil)
+        super(query, options: options)
 
-        @api_key = kwargs[:api_key] || Mihari.config.virustotal_api_key
+        @api_key = api_key || Mihari.config.virustotal_api_key
       end
 
       def artifacts
@@ -53,13 +46,14 @@ module Mihari
         responses = []
 
         loop do
-          response = Structs::VirusTotalIntelligence::Response.from_dynamic!(client.intel_search(query, cursor: cursor))
+          response = Structs::VirusTotalIntelligence::Response.from_dynamic!(client.intel_search(query,
+            cursor: cursor))
           responses << response
           break if response.meta.cursor.nil?
 
           cursor = response.meta.cursor
           # sleep #{interval} seconds to avoid the rate limitation (if it is set)
-          sleep interval
+          sleep(interval) if interval
         end
 
         responses

data/lib/mihari/analyzers/zoomeye.rb

@@ -3,28 +3,23 @@
 module Mihari
   module Analyzers
     class ZoomEye < Base
-      param :query
-
-      option :type, default: proc { "host" }
-
-      option :interval, default: proc { 0 }
-
       # @return [String, nil]
       attr_reader :api_key
 
-      # @return [String]
-      attr_reader :query
-
       # @return [String]
       attr_reader :type
 
-      # @return [Integer]
-      attr_reader :interval
-
-      def initialize(*args, **kwargs)
-        super(*args, **kwargs)
+      #
+      # @param [String] query
+      # @param [Hash, nil] options
+      # @param [String, nil] api_key
+      # @param [String] type
+      #
+      def initialize(query, options: nil, api_key: nil, type: "host")
+        super(query, options: options)
 
-        @api_key = kwargs[:api_key] || Mihari.config.zoomeye_api_key
+        @type = type
+        @api_key = api_key || Mihari.config.zoomeye_api_key
       end
 
       def artifacts
@@ -109,7 +104,7 @@ module Mihari
           break if total <= page * PAGE_SIZE
 
           # sleep #{interval} seconds to avoid the rate limitation (if it is set)
-          sleep interval
+          sleep(interval) if interval
         end
         convert_responses responses.compact
       end
@@ -142,7 +137,7 @@ module Mihari
           break if total <= page * PAGE_SIZE
 
           # sleep #{interval} seconds to avoid the rate limitation (if it is set)
-          sleep interval
+          sleep(interval) if interval
         end
         convert_responses responses.compact
       end

data/lib/mihari/commands/search.rb

@@ -13,7 +13,7 @@ module Mihari
           # @return [Boolean]
           attr_reader :force_overwrite
 
-          def initialize(rule:, force_overwrite:)
+          def initialize(rule, force_overwrite:)
             @rule = rule
             @force_overwrite = force_overwrite
           end
@@ -37,14 +37,23 @@ module Mihari
           end
 
           def run
+            begin
+              analyzer = rule.analyzer
+            rescue ConfigurationError => e
+              # if there is a configuration error, output that error without the stack trace
+              Mihari.logger.error e.to_s
+              return
+            end
+
             with_error_notification do
-              alert = rule.analyzer.run
-              if alert
-                data = Mihari::Entities::Alert.represent(alert)
-                puts JSON.pretty_generate(data.as_json)
-              else
+              alert = analyzer.run
+              if alert.nil?
                 Mihari.logger.info "There is no new artifact found"
+                return
               end
+
+              data = Mihari::Entities::Alert.represent(alert)
+              puts JSON.pretty_generate(data.as_json)
             end
           end
         end
@@ -69,7 +78,7 @@ module Mihari
                 end
 
                 force_overwrite = options["force_overwrite"] || false
-                wrapper = RuleWrapper.new(rule: rule, force_overwrite: force_overwrite)
+                wrapper = RuleWrapper.new(rule, force_overwrite: force_overwrite)
 
                 if wrapper.diff? && !force_overwrite
                   message = "There is diff in the rule (#{rule.id}). Are you sure you want to overwrite the rule? (y/n)"

data/lib/mihari/config.rb

@@ -0,0 +1,133 @@
+module Mihari
+  class Config
+    # @return [String, nil]
+    attr_accessor :binaryedge_api_key
+
+    # @return [String, nil]
+    attr_accessor :censys_id
+
+    # @return [String, nil]
+    attr_accessor :censys_secret
+
+    # @return [String, nil]
+    attr_accessor :circl_passive_password
+
+    # @return [String, nil]
+    attr_accessor :circl_passive_username
+
+    # @return [URI]
+    attr_accessor :database_url
+
+    # @return [String, nil]
+    attr_accessor :greynoise_api_key
+
+    # @return [String, nil]
+    attr_accessor :ipinfo_api_key
+
+    # @return [String, nil]
+    attr_accessor :misp_url
+
+    # @return [String, nil]
+    attr_accessor :misp_api_key
+
+    # @return [String, nil]
+    attr_accessor :onyphe_api_key
+
+    # @return [String, nil]
+    attr_accessor :otx_api_key
+
+    # @return [String, nil]
+    attr_accessor :passivetotal_api_key
+
+    # @return [String, nil]
+    attr_accessor :passivetotal_username
+
+    # @return [String, nil]
+    attr_accessor :pulsedive_api_key
+
+    # @return [String, nil]
+    attr_accessor :securitytrails_api_key
+
+    # @return [String, nil]
+    attr_accessor :shodan_api_key
+
+    # @return [String, nil]
+    attr_accessor :slack_channel
+
+    # @return [String, nil]
+    attr_accessor :slack_webhook_url
+
+    # @return [String, nil]
+    attr_accessor :thehive_url
+
+    # @return [String, nil]
+    attr_accessor :thehive_api_key
+
+    # @return [String, nil]
+    attr_accessor :thehive_api_version
+
+    # @return [String, nil]
+    attr_accessor :urlscan_api_key
+
+    # @return [String, nil]
+    attr_accessor :virustotal_api_key
+
+    # @return [String, nil]
+    attr_accessor :zoomeye_api_key
+
+    # @return [String, nil]
+    attr_accessor :sentry_dsn
+
+    # @return [String, nil]
+    attr_accessor :hide_config_values
+
+    def initialize
+      @binaryedge_api_key = ENV.fetch("BINARYEDGE_API_KEY", nil)
+
+      @censys_id = ENV.fetch("CENSYS_ID", nil)
+      @censys_secret = ENV.fetch("CENSYS_SECRET", nil)
+
+      @circl_passive_password = ENV.fetch("CIRCL_PASSIVE_PASSWORD", nil)
+      @circl_passive_username = ENV.fetch("CIRCL_PASSIVE_USERNAME", nil)
+
+      @database_url = URI(ENV.fetch("DATABASE_URL", "sqlite3:///mihari.db"))
+
+      @greynoise_api_key = ENV.fetch("GREYNOISE_API_KEY", nil)
+
+      @ipinfo_api_key = ENV.fetch("IPINFO_API_KEY", nil)
+
+      @misp_url = ENV.fetch("MISP_URL", nil)
+      @misp_api_key = ENV.fetch("MISP_API_KEY", nil)
+
+      @onyphe_api_key = ENV.fetch("ONYPHE_API_KEY", nil)
+
+      @otx_api_key = ENV.fetch("OTX_API_KEY", nil)
+
+      @passivetotal_api_key = ENV.fetch("PASSIVETOTAL_API_KEY", nil)
+      @passivetotal_username = ENV.fetch("PASSIVETOTAL_USERNAME", nil)
+
+      @pulsedive_api_key = ENV.fetch("PULSEDIVE_API_KEY", nil)
+
+      @securitytrails_api_key = ENV.fetch("SECURITYTRAILS_API_KEY", nil)
+
+      @shodan_api_key = ENV.fetch("SHODAN_API_KEY", nil)
+
+      @slack_channel = ENV.fetch("SLACK_CHANNEL", nil)
+      @slack_webhook_url = ENV.fetch("SLACK_WEBHOOK_URL", nil)
+
+      @thehive_url = ENV.fetch("THEHIVE_URL", nil)
+      @thehive_api_key = ENV.fetch("THEHIVE_API_KEY", nil)
+      @thehive_api_version = ENV.fetch("THEHIVE_API_VERSION", nil)
+
+      @urlscan_api_key = ENV.fetch("URLSCAN_API_KEY", nil)
+
+      @virustotal_api_key = ENV.fetch("VIRUSTOTAL_API_KEY", nil)
+
+      @zoomeye_api_key = ENV.fetch("ZOOMEYE_API_KEY", nil)
+
+      @sentry_dsn = ENV.fetch("SENTRY_DSN", nil)
+
+      @hide_config_values = ENV.fetch("HIDE_CONFIG_VALUES", false)
+    end
+  end
+end

data/lib/mihari/constants.rb

@@ -6,4 +6,7 @@ module Mihari
   DEFAULT_EMITTERS = %w[database misp slack the_hive].map { |name| { emitter: name } }.freeze
 
   DEFAULT_ENRICHERS = %w[whois ipinfo shodan google_public_dns].map { |name| { enricher: name } }.freeze
+
+  DEFAULT_RETRY_TIMES = 3
+  DEFAULT_RETRY_INTERVAL = 5
 end

data/lib/mihari/emitters/slack.rb

@@ -8,10 +8,20 @@ module Mihari
     class Attachment
       include Memist::Memoizable
 
-      extend Dry::Initializer
+      # @return [String]
+      attr_reader :data
+
+      # @return [String]
+      attr_reader :data_type
 
-      option :data
-      option :data_type
+      #
+      # @param [String] data
+      # @param [String] data_type
+      #
+      def initialize(data:, data_type:)
+        @data = data
+        @data_type = data_type
+      end
 
       def actions
         [vt_link, urlscan_link, censys_link, shodan_link].compact

data/lib/mihari/entities/rule.rb

@@ -31,7 +31,7 @@ module Mihari
     end
 
     class RuleIDs < Grape::Entity
-      expose :rule_ids, documentation: { type: Array[String], required: true }, as: :ruleIds
+      expose :rule_ids, documentation: { type: [String], required: true }, as: :ruleIds
     end
   end
 end

data/lib/mihari/entities/tag.rb

@@ -7,7 +7,7 @@ module Mihari
     end
 
     class Tags < Grape::Entity
-      expose :tags, documentation: { type: Array[String], required: true }
+      expose :tags, documentation: { type: [String], required: true }
     end
   end
 end

data/lib/mihari/errors.rb

@@ -21,7 +21,7 @@ module Mihari
 
   class HTTPError < Error; end
 
-  class UnsuccessfulStatusCodeError < HTTPError; end
+  class StatusCodeError < HTTPError; end
 
   class NetworkError < HTTPError; end
 

data/lib/mihari/http.rb

@@ -93,9 +93,8 @@ module Mihari
     def request(req)
       Net::HTTP.start(url.host, url.port, https_options) do |http|
         res = http.request(req)
-        unless res.is_a?(Net::HTTPSuccess)
-          raise UnsuccessfulStatusCodeError, "Unsuccessful response code returned: #{res.code}"
-        end
+
+        raise StatusCodeError, "Unsuccessful response code returned: #{res.code}" unless res.is_a?(Net::HTTPSuccess)
 
         res
       end

data/lib/mihari/schemas/analyzer.rb

@@ -4,10 +4,12 @@ module Mihari
   module Schemas
     AnalyzerOptions = Dry::Schema.Params do
       optional(:interval).value(:integer)
+      optional(:retry_times).value(:integer).default(DEFAULT_RETRY_TIMES)
+      optional(:retry_interval).value(:integer).default(DEFAULT_RETRY_INTERVAL)
     end
 
     AnalyzerWithoutAPIKey = Dry::Schema.Params do
-      required(:analyzer).value(Types::String.enum("crtsh", "dnpedia", "dnstwister"))
+      required(:analyzer).value(Types::String.enum("dnstwister"))
       required(:query).value(:string)
       optional(:options).hash(AnalyzerOptions)
     end
@@ -23,6 +25,7 @@ module Mihari
           "securitytrails",
           "shodan",
           "st",
+          "urlscan",
           "virustotal_intelligence",
           "virustotal",
           "vt_intel",
@@ -72,12 +75,6 @@ module Mihari
       optional(:options).hash(AnalyzerOptions)
     end
 
-    Urlscan = Dry::Schema.Params do
-      required(:analyzer).value(Types::String.enum("urlscan"))
-      required(:query).value(:string)
-      optional(:options).hash(AnalyzerOptions)
-    end
-
     Feed = Dry::Schema.Params do
       required(:analyzer).value(Types::String.enum("feed"))
       required(:query).value(:string)

data/lib/mihari/schemas/rule.rb

@@ -22,7 +22,7 @@ module Mihari
       optional(:updated_on).value(:date)
 
       required(:queries).value(:array).each do
-        AnalyzerWithoutAPIKey | AnalyzerWithAPIKey | Censys | CIRCL | PassiveTotal | ZoomEye | Urlscan | Crtsh | Feed
+        AnalyzerWithoutAPIKey | AnalyzerWithAPIKey | Censys | CIRCL | PassiveTotal | ZoomEye | Crtsh | Feed
       end
 
       optional(:emitters).value(:array).each { Database | MISP | TheHive | Slack | Webhook }.default(DEFAULT_EMITTERS)

data/lib/mihari/structs/config.rb

@@ -38,30 +38,53 @@ module Mihari
 
       class << self
         #
-        # @param [Class<Mihari::Analyzers::Base>, Class<Mihari::Emitters::Base>] klass
+        # Get a type of a class
+        #
+        # @param [Class<Mihari::Analyzers::Base>, Class<Mihari::Emitters::Base>, Class<Mihari::Enrichers::Base>] klass
+        #
+        # @return [String, nil]
+        #
+        def get_type(klass)
+          return "Analyzer" if klass.ancestors.include?(Mihari::Analyzers::Base)
+          return "Emitter" if klass.ancestors.include?(Mihari::Emitters::Base)
+          return "Enricher" if klass.ancestors.include?(Mihari::Enrichers::Base)
+
+          nil
+        end
+
+        #
+        # Get a dummy instance
+        #
+        # @param [Class<Mihari::Analyzers::Base>, Class<Mihari::Emitters::Base>, Class<Mihari::Enrichers::Base>] klass
+        #
+        # @return [Mihari::Analyzers::Base, Mihari::Emitter::Base, Mihari::Enricher::Base] dummy
+        #
+        def get_dummy(klass)
+          type = get_type(klass)
+          is_analyzer = type == "Analyzer"
+          is_analyzer ? klass.new("dummy") : klass.new(artifacts: [], rule: nil)
+        end
+
+        #
+        # @param [Class<Mihari::Analyzers::Base>, Class<Mihari::Emitters::Base>, Class<Mihari::Enrichers::Base>] klass
         #
         # @return [Mihari::Structs::Config, nil] config
         #
         def from_class(klass)
           return nil if klass == Mihari::Analyzers::Rule
 
-          name = klass.to_s.split("::").last.to_s
-
-          is_analyzer = klass.ancestors.include?(Mihari::Analyzers::Base)
-          is_emitter = klass.ancestors.include?(Mihari::Emitters::Base)
-          is_enricher = klass.ancestors.include?(Mihari::Enrichers::Base)
-
-          type = "Analyzer"
-          type = "Emitter" if is_emitter
-          type = "Enricher" if is_enricher
+          type = get_type(klass)
+          return nil if type.nil?
 
           begin
-            instance = is_analyzer ? klass.new("dummy") : klass.new(artifacts: [], rule: nil)
-            is_configured = instance.configured?
-            values = instance.configuration_values
-
-            new(name: name, values: values, is_configured: is_configured, type: type)
-          rescue ArgumentError => _e
+            instance = get_dummy(klass)
+            new(
+              name: klass.to_s.split("::").last.to_s,
+              values: instance.configuration_values,
+              is_configured: instance.configured?,
+              type: type
+            )
+          rescue ArgumentError
             nil
           end
         end

data/lib/mihari/structs/rule.rb

@@ -162,7 +162,7 @@ module Mihari
       # @return [Mihari::Analyzers::Rule]
       #
       def analyzer
-        Mihari::Analyzers::Rule.new(rule: self)
+        Mihari::Analyzers::Rule.new(self)
       end
 
       class << self

data/lib/mihari/type_checker.rb

@@ -2,13 +2,13 @@
 
 module Mihari
   class TypeChecker
-    extend Dry::Initializer
-
-    param :data
-
-    def initialize(*args, **kwargs)
-      super
+    # @return [String]
+    attr_reader :data
 
+    #
+    # @param [String] data
+    #
+    def initialize(data)
       raise ArgumentError if data.is_a?(Hash)
 
       @data = data.to_s

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "5.2.3"
+  VERSION = "5.3.0"
 end