mihari 4.1.1 → 4.3.0

data/lib/mihari/{web/entities → entities}/rule.rb

@@ -8,6 +8,10 @@ module Mihari
       expose :interval, documentation: { type: Integer, required: false }
     end
 
+    class Emitter < Grape::Entity
+      expose :emitter, documentation: { type: String, required: true }
+    end
+
     class Rule < Grape::Entity
       expose :id, documentation: { type: String, required: true }
 
@@ -16,6 +20,7 @@ module Mihari
       expose :title, documentation: { type: String, required: true }
       expose :description, documentation: { type: String, required: true }
       expose :queries, using: Entities::Query, documentation: { type: Entities::Query, is_array: true, required: true }
+      expose :emitters, using: Entities::Emitter, documentation: { type: Entities::Emitter, is_array: true, required: false }
       expose :tags, documentation: { type: String, is_array: true }
       expose :allowed_data_types, documentation: { type: String, is_array: true }, as: :allowedDtaTypes
       expose :disallowed_data_values, documentation: { type: String, is_array: true }, as: :disallowedDataValues

data/lib/mihari/errors.rb

@@ -14,4 +14,6 @@ module Mihari
   class FeedParseError < Error; end
 
   class RuleValidationError < Error; end
+
+  class ConfigurationError < Error; end
 end

data/lib/mihari/feed/reader.rb

@@ -1,46 +1,40 @@
 # frozen_string_literal: true
 
 require "csv"
+require "insensitive_hash"
 
 module Mihari
   module Feed
     class Reader
-      attr_reader :uri, :http_request_headers, :http_request_method, :http_request_payload_type, :http_request_payload
+      attr_reader :uri, :http_request_headers, :http_request_method, :http_request_payload
 
       def initialize(uri, http_request_headers: {}, http_request_method: "GET", http_request_payload_type: nil, http_request_payload: {})
         @uri = Addressable::URI.parse(uri)
-        @http_request_headers = http_request_headers
+        @http_request_headers = http_request_headers.insensitive
         @http_request_method = http_request_method
-        @http_request_payload_type = http_request_payload_type
         @http_request_payload = http_request_payload
+
+        http_request_headers["content-type"] = http_request_payload_type if http_request_payload_type
       end
 
       def read
         return read_file(uri.path) if uri.scheme == "file"
 
-        return get if http_request_method == "GET"
-
-        post
-      end
-
-      def get
-        uri.query = Addressable::URI.form_encode(http_request_payload)
-        get = Net::HTTP::Get.new(uri)
+        res = nil
+        client = HTTP.new(uri, headers: http_request_headers, payload: http_request_payload)
 
-        request(get)
-      end
+        res = client.get if http_request_method == "GET"
+        res = client.post if http_request_method == "POST"
 
-      def post
-        post = Net::HTTP::Post.new(uri)
+        return [] if res.nil?
 
-        case http_request_payload_type
-        when "application/json"
-          post.body = JSON.generate(http_request_payload)
-        when "application/x-www-form-urlencoded"
-          post.set_form_data(http_request_payload)
+        body = res.body
+        content_type = res["Content-Type"].to_s
+        if content_type.include?("application/json")
+          convert_as_json(body)
+        else
+          convert_as_csv(body)
         end
-
-        request(post)
       end
 
       #
@@ -70,42 +64,6 @@ module Mihari
         CSV.new(text_without_comments).to_a.reject(&:empty?)
       end
 
-      def https_options
-        return { use_ssl: true } if uri.scheme == "https"
-
-        {}
-      end
-
-      #
-      # Make a HTTP request
-      #
-      # @param [Net::HTTPRequest] req
-      #
-      # @return [Array<Hash>]
-      #
-      def request(req)
-        Net::HTTP.start(uri.host, uri.port, https_options) do |http|
-          # set headers
-          http_request_headers.each do |k, v|
-            req[k] = v
-          end
-
-          response = http.request(req)
-
-          code = response.code.to_i
-          raise HttpError, "Unsupported response code returned: #{code}" if code != 200
-
-          body = response.body
-
-          content_type = response["Content-Type"].to_s
-          if content_type.include?("application/json")
-            convert_as_json(body)
-          else
-            convert_as_csv(body)
-          end
-        end
-      end
-
       #
       # Read & convert a file
       #

data/lib/mihari/http.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+require "insensitive_hash"
+
+module Mihari
+  class HTTP
+    attr_reader :uri, :headers, :payload
+
+    def initialize(uri, headers: {}, payload: {})
+      @uri = uri.is_a?(Addressable::URI) ? uri : Addressable::URI.parse(uri)
+      @headers = headers.insensitive
+      @payload = payload
+    end
+
+    #
+    # Make a GET request
+    #
+    # @return [Net::HTTPResponse]
+    #
+    def get
+      uri.query = Addressable::URI.form_encode(payload)
+      get = Net::HTTP::Get.new(uri)
+
+      request(get)
+    end
+
+    #
+    # Make a POST request
+    #
+    # @return [Net::HTTPResponse]
+    #
+    def post
+      post = Net::HTTP::Post.new(uri)
+
+      case content_type
+      when "application/json"
+        post.body = JSON.generate(payload)
+      when "application/x-www-form-urlencoded"
+        post.set_form_data(payload)
+      end
+
+      request(post)
+    end
+
+    class << self
+      def get(uri, headers: {}, payload: {})
+        client = new(uri, headers: headers, payload: payload)
+        client.get
+      end
+
+      def post(uri, headers: {}, payload: {})
+        client = new(uri, headers: headers, payload: payload)
+        client.post
+      end
+    end
+
+    private
+
+    def content_type
+      headers["content-type"] || "application/json"
+    end
+
+    #
+    # Get options for HTTP request
+    #
+    # @return [Hahs]
+    #
+    def https_options
+      return { use_ssl: true } if uri.scheme == "https"
+
+      {}
+    end
+
+    #
+    # Make a HTTP request
+    #
+    # @param [Net::HTTPRequest] req
+    #
+    # @return [Net::HTTPResponse]
+    #
+    def request(req)
+      Net::HTTP.start(uri.host, uri.port, https_options) do |http|
+        # set headers
+        headers.each do |k, v|
+          req[k] = v
+        end
+
+        res = http.request(req)
+
+        unless res.is_a?(Net::HTTPSuccess)
+          code = res.code.to_i
+          raise HttpError, "Unsuccessful response code returned: #{code}"
+        end
+
+        res
+      end
+    end
+  end
+end

data/lib/mihari/mixins/error_notification.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Mixins
+    module ErrorNotification
+      #
+      # Send an exception notification if there is any error in a block
+      #
+      # @return [Nil]
+      #
+      def with_error_notification
+        yield
+      rescue StandardError => e
+        Mihari.logger.error e
+
+        Sentry.capture_exception(e) if Sentry.initialized?
+      end
+    end
+  end
+end

data/lib/mihari/mixins/retriable.rb

@@ -3,20 +3,30 @@
 module Mihari
   module Mixins
     module Retriable
+      DEFAULT_ON = [
+        Errno::ECONNRESET,
+        Errno::ECONNABORTED,
+        Errno::EPIPE,
+        OpenSSL::SSL::SSLError,
+        Timeout::Error,
+        RetryableError
+      ]
+
       #
       # Retry on error
       #
       # @param [Integer] times
       # @param [Integer] interval
+      # @param [Array<StandardError>] on
       #
       # @return [nil]
       #
-      def retry_on_error(times: 3, interval: 10)
+      def retry_on_error(times: 3, interval: 5, on: DEFAULT_ON)
         try = 0
         begin
           try += 1
           yield
-        rescue Errno::ECONNRESET, Errno::ECONNABORTED, Errno::EPIPE, OpenSSL::SSL::SSLError, Timeout::Error, RetryableError => e
+        rescue *on => e
           sleep interval
           retry if try < times
           raise e

data/lib/mihari/mixins/rule.rb

@@ -48,8 +48,7 @@ module Mihari
       def validate_rule!(rule)
         rule.validate!
       rescue RuleValidationError => e
-        error_message = "Failed to parse the input as a rule:\n#{e.message}"
-        puts error_message.colorize(:red)
+        Mihari.logger.error "Failed to parse the input as a rule"
         raise e
       end
 

data/lib/mihari/schemas/rule.rb

@@ -42,14 +42,26 @@ module Mihari
     Feed = Dry::Schema.Params do
       required(:analyzer).value(Types::String.enum("feed"))
       required(:query).value(:string)
-      required(:http_request_method).value(Types::FeedHttpRequestMethods).default("GET")
-      required(:http_request_headers).value(:hash).default({})
-      required(:http_request_payload).value(:hash).default({})
       required(:selector).value(:string)
-      optional(:http_request_payload_type).value(Types::FeedHttpRequestPayloadTypes)
+      optional(:http_request_method).value(Types::HttpRequestMethods).default("GET")
+      optional(:http_request_headers).value(:hash).default({})
+      optional(:http_request_payload).value(:hash).default({})
+      optional(:http_request_payload_type).value(Types::HttpRequestPayloadTypes)
       optional(:options).hash(AnalyzerOptions)
     end
 
+    Emitter = Dry::Schema.Params do
+      required(:emitter).value(Types::EmitterTypes)
+    end
+
+    HTTP = Dry::Schema.Params do
+      required(:emitter).value(Types::String.enum("http"))
+      required(:url).value(:string)
+      optional(:http_request_method).value(Types::HttpRequestMethods).default("POST")
+      optional(:http_request_headers).value(:hash).default({})
+      optional(:template).value(:string)
+    end
+
     Rule = Dry::Schema.Params do
       required(:title).value(:string)
       required(:description).value(:string)
@@ -63,11 +75,25 @@ module Mihari
 
       required(:queries).value(:array).each { Analyzer | Spyse | ZoomEye | Urlscan | Crtsh | Feed }
 
+      optional(:emitters).value(:array).each { Emitter | HTTP }
+
       optional(:allowed_data_types).value(array[Types::DataTypes]).default(ALLOWED_DATA_TYPES)
       optional(:disallowed_data_values).value(array[:string]).default([])
 
       optional(:ignore_old_artifacts).value(:bool).default(false)
       optional(:ignore_threshold).value(:integer).default(0)
+
+      before(:key_coercer) do |result|
+        # it looks like that dry-schema v1.9.1 has an issue with setting an array of schemas as a default value
+        # e.g. optional(:emitters).value(:array).each { Emitter | HTTP }.default(DEFAULT_EMITTERS) does not work well
+        # so let's do a dirty hack...
+        h = result.to_h
+
+        emitters = h[:emitters]
+        h[:emitters] = emitters || DEFAULT_EMITTERS
+
+        h
+      end
     end
 
     class RuleContract < Dry::Validation::Contract

data/lib/mihari/structs/ipinfo.rb

@@ -14,13 +14,12 @@ module Mihari
           include Mixins::AutonomousSystem
 
           def from_dynamic!(d)
+            d = d.deep_stringify_keys
             d = Types::Hash[d]
 
             asn = nil
             asn_ = d.dig("asn", "asn")
-            unless asn_.nil?
-              asn = normalize_asn(asn_)
-            end
+            asn = normalize_asn(asn_) unless asn_.nil?
 
             new(
               ip: d.fetch("ip"),

data/lib/mihari/structs/rule.rb

@@ -121,6 +121,37 @@ module Mihari
             data: data
           )
         end
+
+        #
+        # @return [Mihari::Analyzers::Rule]
+        #
+        def to_analyzer
+          analyzer = Mihari::Analyzers::Rule.new(
+            title: self[:title],
+            description: self[:description],
+            tags: self[:tags],
+            queries: self[:queries],
+            allowed_data_types: self[:allowed_data_types],
+            disallowed_data_values: self[:disallowed_data_values],
+            emitters: self[:emitters],
+            id: id
+          )
+          analyzer.ignore_old_artifacts = self[:ignore_old_artifacts]
+          analyzer.ignore_threshold = self[:ignore_threshold]
+
+          analyzer
+        end
+
+        class << self
+          #
+          # @param [Mihari::Rule] model
+          #
+          # @return [Mihari::Structs::Rule::Rule]
+          #
+          def from_model(model)
+            Structs::Rule::Rule.new(model.data)
+          end
+        end
       end
     end
   end

data/lib/mihari/structs/shodan.rb

@@ -26,9 +26,17 @@ module Mihari
 
         def self.from_dynamic!(d)
           d = Types::Hash[d]
+
+          # hostnames should be an array of string but sometimes Shodan returns a string
+          # e.g. "hostnames": "set(['149.28.146.131.vultr.com', 'rebs.ga'])",
+          # https://github.com/ninoseki/mihari/issues/424
+          # so use an empty array if hostnames is a string
+          hostnames = d.fetch("hostnames")
+          hostnames = [] if hostnames.is_a?(String)
+
           new(
             asn: d["asn"],
-            hostnames: d.fetch("hostnames"),
+            hostnames: hostnames,
             location: Location.from_dynamic!(d.fetch("location")),
             domains: d.fetch("domains"),
             ip_str: d.fetch("ip_str"),

data/lib/mihari/types.rb

@@ -22,7 +22,6 @@ module Mihari
       "circl",
       "dnpedia",
       "dnstwister",
-      "feed",
       "greynoise",
       "onyphe",
       "otx",
@@ -38,7 +37,16 @@ module Mihari
       "vt"
     )
 
-    FeedHttpRequestMethods = Types::String.enum("GET", "POST")
-    FeedHttpRequestPayloadTypes = Types::String.enum("application/json", "application/x-www-form-urlencoded")
+    HttpRequestMethods = Types::String.enum("GET", "POST")
+    HttpRequestPayloadTypes = Types::String.enum("application/json", "application/x-www-form-urlencoded")
+
+    EmitterTypes = Types::String.enum(
+      "database",
+      "http",
+      "misp",
+      "slack",
+      "the_hive",
+      "webhook"
+    )
   end
 end

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "4.1.1"
+  VERSION = "4.3.0"
 end

data/lib/mihari/web/api.rb

@@ -1,25 +1,5 @@
 # frozen_string_literal: true
 
-# Entities
-require "mihari/web/entities/message"
-
-require "mihari/web/entities/autonomous_system"
-require "mihari/web/entities/command"
-require "mihari/web/entities/config"
-require "mihari/web/entities/dns"
-require "mihari/web/entities/geolocation"
-require "mihari/web/entities/ip_address"
-require "mihari/web/entities/reverse_dns"
-require "mihari/web/entities/source"
-require "mihari/web/entities/tag"
-require "mihari/web/entities/whois"
-
-require "mihari/web/entities/artifact"
-
-require "mihari/web/entities/alert"
-
-require "mihari/web/entities/rule"
-
 # Endpoints
 require "mihari/web/endpoints/alerts"
 require "mihari/web/endpoints/artifacts"

data/lib/mihari/web/app.rb

@@ -6,12 +6,11 @@ require "rack/contrib"
 require "rack/handler/puma"
 require "rack/cors"
 
-require "grape"
-require "grape-entity"
 require "grape-swagger"
 require "grape-swagger-entity"
 
 require "mihari/web/middleware/connection_adapter"
+require "mihari/web/middleware/error_notification_adapter"
 
 require "mihari/web/api"
 
@@ -37,6 +36,7 @@ module Mihari
           end
 
           use Middleware::ConnectionAdapter
+          use Middleware::ErrorNotificationAdapter
 
           run App.new
         end.to_app

data/lib/mihari/web/endpoints/rules.rb

@@ -79,7 +79,9 @@ module Mihari
             error!({ message: "ID:#{id} is not found" }, 404)
           end
 
-          Mihari::CLI::Main.start ["search", rule.id]
+          struct = Mihari::Structs::Rule::Rule.from_model(rule)
+          analyzer = struct.to_analyzer
+          analyzer.run
 
           status 201
           present({ message: "ID:#{id} is ran successfully" }, with: Entities::Message)

data/lib/mihari/web/middleware/error_notification_adapter.rb

@@ -0,0 +1,19 @@
+module Mihari
+  module Middleware
+    class ErrorNotificationAdapter
+      include Mixins::ErrorNotification
+
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        with_error_notification do
+          status, headers, body = @app.call(env)
+
+          [status, headers, body]
+        end
+      end
+    end
+  end
+end

data/lib/mihari/web/public/index.html

@@ -1 +1 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1"><link rel="icon" href="/static/favicon.ico"><title>Mihari</title><link href="/static/js/app.b88ce341.js" rel="preload" as="script"></head><body><noscript><strong>We're sorry but Mihari doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id="app"></div><script src="/static/js/app.b88ce341.js"></script></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><meta name="viewport" content="width=device-width,initial-scale=1"/><link rel="icon" href="/static/favicon.ico"/><title>Mihari</title><script defer="defer" type="module" src="/static/js/chunk-vendors.c5525f1e.js"></script><script defer="defer" type="module" src="/static/js/app.e74e91d7.js"></script><link href="/static/css/chunk-vendors.c57bb3fd.css" rel="stylesheet"><link href="/static/css/app.0de4b715.css" rel="stylesheet"><script defer="defer" src="/static/js/chunk-vendors-legacy.41357cdf.js" nomodule></script><script defer="defer" src="/static/js/app-legacy.e451304b.js" nomodule></script></head><body><noscript><strong>We're sorry but Mihari doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id="app"></div></body></html>