mihari 5.4.1 → 5.4.3

data/lib/mihari/clients/securitytrails.rb

@@ -15,6 +15,50 @@ module Mihari
         super(base_url, headers: headers)
       end
 
+      #
+      # Domain search
+      #
+      # @param [String] query
+      #
+      # @return [Array<String>]
+      #
+      def domain_search(query)
+        records = get_all_dns_history(query, type: "a")
+        records.map do |record|
+          (record["values"] || []).map { |value| value["ip"] }
+        end.flatten.compact.uniq
+      end
+
+      #
+      # IP search
+      #
+      # @param [String] query
+      #
+      # @return [Array<Mihari::Artifact>]
+      #
+      def ip_search(query)
+        records = search_by_ip(query)
+        records.filter_map do |record|
+          data = record["hostname"]
+          Artifact.new(data: data, metadata: record)
+        end
+      end
+
+      #
+      # Mail search
+      #
+      # @param [String] query
+      #
+      # @return [Array<String>]
+      #
+      def mail_search(query)
+        records = search_by_mail(query)
+        records.filter_map do |record|
+          data = record["hostname"]
+          Artifact.new(data: data, metadata: record)
+        end
+      end
+
       #
       # @param [String] mail
       #

data/lib/mihari/clients/shodan.rb

@@ -3,6 +3,8 @@
 module Mihari
   module Clients
     class Shodan < Base
+      PAGE_SIZE = 100
+
       # @return [String]
       attr_reader :api_key
 
@@ -10,11 +12,12 @@ module Mihari
       # @param [String] base_url
       # @param [String, nil] api_key
       # @param [Hash] headers
+      # @param [Integer, nil] interval
       #
-      def initialize(base_url = "https://api.shodan.io", api_key:, headers: {})
+      def initialize(base_url = "https://api.shodan.io", api_key:, headers: {}, interval: nil)
         raise(ArgumentError, "'api_key' argument is required") unless api_key
 
-        super(base_url, headers: headers)
+        super(base_url, headers: headers, interval: interval)
 
         @api_key = api_key
       end
@@ -36,6 +39,31 @@ module Mihari
         res = get("/shodan/host/search", params: params)
         Structs::Shodan::Result.from_dynamic! JSON.parse(res.body.to_s)
       end
+
+      #
+      # @param [String] query
+      # @param [Boolean] minify
+      # @param [Integer] pagination_limit
+      #
+      # @return [Enumerable<Structs::Shodan::Result>]
+      #
+      def search_with_pagination(query, minify: true, pagination_limit: Mihari.config.pagination_limit)
+        Enumerator.new do |y|
+          (1..pagination_limit).each do |page|
+            res = search(query, page: page, minify: minify)
+
+            y.yield res
+
+            break if res.total <= page * PAGE_SIZE
+
+            sleep_interval
+          rescue JSON::ParserError
+            # ignore JSON::ParserError
+            # ref. https://github.com/ninoseki/mihari/issues/197
+            next
+          end
+        end
+      end
     end
   end
 end

data/lib/mihari/clients/urlscan.rb

@@ -7,26 +7,52 @@ module Mihari
       # @param [String] base_url
       # @param [String, nil] api_key
       # @param [Hash] headers
+      # @param [Interval, nil] interval
       #
-      def initialize(base_url = "https://urlscan.io", api_key:, headers: {})
+      def initialize(base_url = "https://urlscan.io", api_key:, headers: {}, interval: nil)
         raise(ArgumentError, "'api_key' argument is required") if api_key.nil?
 
         headers["api-key"] = api_key
 
-        super(base_url, headers: headers)
+        super(base_url, headers: headers, interval: interval)
       end
 
       #
       # @param [String] q
-      # @param [Integer] size
+      # @param [Integer, nil] size
       # @param [String, nil] search_after
       #
-      # @return [Hash]
+      # @return [Structs::Urlscan::Response]
       #
-      def search(q, size: 100, search_after: nil)
+      def search(q, size: nil, search_after: nil)
         params = { q: q, size: size, search_after: search_after }.compact
         res = get("/api/v1/search/", params: params)
-        JSON.parse res.body.to_s
+        Structs::Urlscan::Response.from_dynamic! JSON.parse(res.body.to_s)
+      end
+
+      #
+      # @param [String] q
+      # @param [Integer, nil] size
+      # @param [Integer] pagination_limit
+      #
+      # @return [Enumerable<Structs::Urlscan::Response>]
+      #
+      def search_with_pagination(q, size: nil, pagination_limit: Mihari.config.pagination_limit)
+        search_after = nil
+
+        Enumerator.new do |y|
+          pagination_limit.times do
+            res = search(q, size: size, search_after: search_after)
+
+            y.yield res
+
+            break unless res.has_more
+
+            search_after = res.results.last.sort.join(",")
+
+            sleep_interval
+          end
+        end
       end
     end
   end

data/lib/mihari/clients/virustotal.rb

@@ -7,13 +7,14 @@ module Mihari
       # @param [String] base_url
       # @param [String, nil] api_key
       # @param [Hash] headers
+      # @param [Integer, nil] interval
       #
-      def initialize(base_url = "https://www.virustotal.com", api_key:, headers: {})
+      def initialize(base_url = "https://www.virustotal.com", api_key:, headers: {}, interval: nil)
         raise(ArgumentError, "'api_key' argument is required") if api_key.nil?
 
         headers["x-apikey"] = api_key
 
-        super(base_url, headers: headers)
+        super(base_url, headers: headers, interval: interval)
       end
 
       #
@@ -38,11 +39,35 @@ module Mihari
       # @param [String] query
       # @param [String, nil] cursor
       #
-      # @return [Hash]
+      # @return [Structs::VirusTotalIntelligence::Response]
       #
       def intel_search(query, cursor: nil)
         params = { query: query, cursor: cursor }.compact
-        _get("/api/v3/intelligence/search", params: params)
+        res = _get("/api/v3/intelligence/search", params: params)
+        Structs::VirusTotalIntelligence::Response.from_dynamic! res
+      end
+
+      #
+      # @param [String] query
+      # @param [Integer] pagination_limit
+      #
+      # @return [Enumerable<Structs::VirusTotalIntelligence::Response>]
+      #
+      def intel_search_with_pagination(query, pagination_limit: Mihari.config.pagination_limit)
+        cursor = nil
+
+        Enumerator.new do |y|
+          pagination_limit.times do
+            res = intel_search(query, cursor: cursor)
+
+            y.yield res
+
+            cursor = res.meta.cursor
+            break if cursor.nil?
+
+            sleep_interval
+          end
+        end
       end
 
       private

data/lib/mihari/clients/zoomeye.rb

@@ -3,18 +3,21 @@
 module Mihari
   module Clients
     class ZoomEye < Base
+      PAGE_SIZE = 10
+
       attr_reader :api_key
 
       #
       # @param [String] base_url
       # @param [String, nil] api_key
       # @param [Hash] headers
+      # @param [Integer, nil] interval
       #
-      def initialize(base_url = "https://api.zoomeye.org", api_key:, headers: {})
+      def initialize(base_url = "https://api.zoomeye.org", api_key:, headers: {}, interval: nil)
         raise(ArgumentError, "'api_key' argument is required") unless api_key
 
         headers["api-key"] = api_key
-        super(base_url, headers: headers)
+        super(base_url, headers: headers, interval: interval)
       end
 
       #
@@ -36,6 +39,30 @@ module Mihari
         _get("/host/search", params: params)
       end
 
+      #
+      # @param [String] query
+      # @param [String, nil] facets
+      # @param [Integer] pagination_limit
+      #
+      # @return [Enumerable<Hash>]
+      #
+      def host_search_with_pagination(query, facets: nil, pagination_limit: Mihari.config.pagination_limit)
+        Enumerator.new do |y|
+          (1..pagination_limit).each do |page|
+            res = host_search(query, facets: facets, page: page)
+
+            break if res.nil?
+
+            y.yield res
+
+            total = res["total"].to_i
+            break if total <= page * PAGE_SIZE
+
+            sleep_interval
+          end
+        end
+      end
+
       #
       # Search the Web technologies
       #
@@ -55,6 +82,30 @@ module Mihari
         _get("/web/search", params: params)
       end
 
+      #
+      # @param [String] query
+      # @param [String, nil] facets
+      # @param [Integer] pagination_limit
+      #
+      # @return [Enumerable<Hash>]
+      #
+      def web_search_with_pagination(query, facets: nil, pagination_limit: Mihari.config.pagination_limit)
+        Enumerator.new do |y|
+          (1..pagination_limit).each do |page|
+            res = web_search(query, facets: facets, page: page)
+
+            break if res.nil?
+
+            y.yield res
+
+            total = res["total"].to_i
+            break if total <= page * PAGE_SIZE
+
+            sleep_interval
+          end
+        end
+      end
+
       private
 
       #

data/lib/mihari/commands/alert.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Commands
+    module Alert
+      class << self
+        def included(thor)
+          thor.class_eval do
+            desc "add [PATH]", "Add an alert"
+            #
+            # @param [String] path
+            #
+            def add(path)
+              Mihari::Database.with_db_connection do
+                proxy = Mihari::Services::AlertProxy.from_path(path)
+                proxy.validate!
+
+                runner = Mihari::Services::AlertRunner.new(proxy)
+
+                begin
+                  alert = runner.run
+                rescue ActiveRecord::RecordNotFound => e
+                  # if there is a ActiveRecord::RecordNotFound, output that error without the stack trace
+                  Mihari.logger.error e.to_s
+                  return
+                end
+
+                if alert.nil?
+                  Mihari.logger.info "There is no new artifact found"
+                  return
+                end
+
+                data = Mihari::Entities::Alert.represent(alert)
+                puts JSON.pretty_generate(data.as_json)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mihari/commands/rule.rb

@@ -15,7 +15,7 @@ module Mihari
             # @param [String] path
             #
             def validate(path)
-              rule = Services::Rule.from_path_or_id(path)
+              rule = Services::RuleProxy.from_path_or_id(path)
 
               begin
                 rule.validate!
@@ -47,7 +47,7 @@ module Mihari
               # @return [Mihari::Services::Rule]
               #
               def rule_template
-                Services::Rule.from_path File.expand_path("../templates/rule.yml.erb", __dir__)
+                Services::RuleProxy.from_path File.expand_path("../templates/rule.yml.erb", __dir__)
               end
 
               #

data/lib/mihari/commands/search.rb

@@ -4,60 +4,6 @@ module Mihari
   module Commands
     module Search
       class << self
-        class RuleWrapper
-          include Mixins::ErrorNotification
-
-          # @return [Nihari::Structs::Rule]
-          attr_reader :rule
-
-          # @return [Boolean]
-          attr_reader :force_overwrite
-
-          def initialize(rule, force_overwrite:)
-            @rule = rule
-            @force_overwrite = force_overwrite
-          end
-
-          def force_overwrite?
-            force_overwrite
-          end
-
-          #
-          # @return [Boolean]
-          #
-          def diff?
-            model = Mihari::Rule.find(rule.id)
-            model.data != rule.data.deep_stringify_keys
-          rescue ActiveRecord::RecordNotFound
-            false
-          end
-
-          def update_or_create
-            rule.to_model.save
-          end
-
-          def run
-            begin
-              analyzer = rule.to_analyzer
-            rescue ConfigurationError => e
-              # if there is a configuration error, output that error without the stack trace
-              Mihari.logger.error e.to_s
-              return
-            end
-
-            with_error_notification do
-              alert = analyzer.run
-              if alert.nil?
-                Mihari.logger.info "There is no new artifact found"
-                return
-              end
-
-              data = Mihari::Entities::Alert.represent(alert)
-              puts JSON.pretty_generate(data.as_json)
-            end
-          end
-        end
-
         def included(thor)
           thor.class_eval do
             desc "search [PATH]", "Search by a rule"
@@ -69,7 +15,7 @@ module Mihari
             #
             def search(path_or_id)
               Mihari::Database.with_db_connection do
-                rule = Services::Rule.from_path_or_id path_or_id
+                rule = Services::RuleProxy.from_path_or_id path_or_id
 
                 begin
                   rule.validate!
@@ -78,15 +24,30 @@ module Mihari
                 end
 
                 force_overwrite = options["force_overwrite"] || false
-                wrapper = RuleWrapper.new(rule, force_overwrite: force_overwrite)
+                runner = Services::RuleRunner.new(rule, force_overwrite: force_overwrite)
 
-                if wrapper.diff? && !force_overwrite
+                if runner.diff? && !force_overwrite
                   message = "There is diff in the rule (#{rule.id}). Are you sure you want to overwrite the rule? (y/n)"
                   return unless yes?(message)
                 end
 
-                wrapper.update_or_create
-                wrapper.run
+                runner.update_or_create
+
+                begin
+                  alert = runner.run
+                rescue ConfigurationError => e
+                  # if there is a configuration error, output that error without the stack trace
+                  Mihari.logger.error e.to_s
+                  return
+                end
+
+                if alert.nil?
+                  Mihari.logger.info "There is no new artifact found"
+                  return
+                end
+
+                data = Mihari::Entities::Alert.represent(alert)
+                puts JSON.pretty_generate(data.as_json)
               end
             end
           end

data/lib/mihari/commands/web.rb

@@ -12,7 +12,7 @@ module Mihari
             method_option :threads, type: :string, default: "0:5", desc: "min:max threads to use"
             method_option :verbose, type: :boolean, default: true, desc: "Report each request"
             method_option :worker_timeout, type: :numeric, default: 60, desc: "Worker timeout value (in seconds)"
-            method_option :hide_config_values, type: :boolean, default: false,
+            method_option :hide_config_values, type: :boolean, default: true,
               desc: "Whether to hide config values or not"
             method_option :open, type: :boolean, default: true, desc: "Whether to open the app in browser or not"
             method_option :rack_env, type: :string, default: "production", desc: "Rack environment"

data/lib/mihari/config.rb

@@ -141,12 +141,12 @@ module Mihari
 
       @sentry_dsn = ENV.fetch("SENTRY_DSN", nil)
 
-      @hide_config_values = ENV.fetch("HIDE_CONFIG_VALUES", false)
+      @hide_config_values = ENV.fetch("HIDE_CONFIG_VALUES", true)
 
       @retry_times = ENV.fetch("RETRY_TIMES", 3).to_i
       @retry_interval = ENV.fetch("RETRY_INTERVAL", 5).to_i
 
-      @pagination_limit = ENV.fetch("PAGINATION_LIMIT", 1000).to_i
+      @pagination_limit = ENV.fetch("PAGINATION_LIMIT", 100).to_i
     end
   end
 end

data/lib/mihari/emitters/base.rb

@@ -14,7 +14,7 @@ module Mihari
 
       #
       # @param [Array<Mihari::Artifact>] artifacts
-      # @param [Mihari::Services::Rule] rule
+      # @param [Mihari::Services::RuleProxy] rule
       # @param [Hash] **_options
       #
       def initialize(artifacts:, rule:, **_options)

data/lib/mihari/emitters/database.rb

@@ -10,10 +10,10 @@ module Mihari
       #
       # Create an alert
       #
-      # @return [Mihari::Alert]
+      # @return [Mihari::Alert, nil]
       #
       def emit
-        return if artifacts.empty?
+        return nil if artifacts.empty?
 
         tags = rule.tags.filter_map { |name| Tag.find_or_create_by(name: name) }.uniq
         taggings = tags.map { |tag| Tagging.new(tag_id: tag.id) }

data/lib/mihari/errors.rb

@@ -15,17 +15,38 @@ module Mihari
 
   class RuleValidationError < Error; end
 
+  class AlertValidationError < Error; end
+
   class YAMLSyntaxError < Error; end
 
   class ConfigurationError < Error; end
 
+  # errors for HTTP interactions
   class HTTPError < Error; end
 
-  class StatusCodeError < HTTPError; end
-
   class NetworkError < HTTPError; end
 
   class TimeoutError < HTTPError; end
 
   class SSLError < HTTPError; end
+
+  class StatusCodeError < HTTPError
+    # @return [Integer]
+    attr_reader :status_code
+
+    # @return [String, nil]
+    attr_reader :body
+
+    #
+    # @param [String] msg
+    # @param [Integer] status_code
+    # @param [String, nil] body
+    #
+    def initialize(msg, status_code, body)
+      super(msg)
+
+      @status_code = status_code
+      @body = body
+    end
+  end
 end

data/lib/mihari/http.rb

@@ -94,7 +94,13 @@ module Mihari
       Net::HTTP.start(url.host, url.port, https_options) do |http|
         res = http.request(req)
 
-        raise StatusCodeError, "Unsuccessful response code returned: #{res.code}" unless res.is_a?(Net::HTTPSuccess)
+        unless res.is_a?(Net::HTTPSuccess)
+          raise StatusCodeError.new(
+            "Unsuccessful response code returned: #{res.code}",
+            res.code.to_i,
+            res.body
+          )
+        end
 
         res
       end

data/lib/mihari/schemas/alert.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Schemas
+    Alert = Dry::Schema.Params do
+      required(:rule_id).value(:string)
+      required(:artifacts).value(array[:string])
+    end
+
+    class AlertContract < Dry::Validation::Contract
+      params(Alert)
+    end
+  end
+end