mihari 4.1.1 → 4.3.0

data/lib/mihari.rb

@@ -1,10 +1,20 @@
 # frozen_string_literal: true
 
+# standard libs
+require "ipaddr"
+require "json"
+require "net/http"
+require "net/https"
+require "resolv"
+require "yaml"
+
+# Active Support & Active Record
 require "active_support"
 
 require "active_support/core_ext/hash"
 require "active_support/core_ext/integer/time"
 require "active_support/core_ext/numeric/time"
+require "active_support/core_ext/object/deep_dup"
 
 require "active_record"
 
@@ -17,23 +27,22 @@ require "dry/struct"
 require "dry/types"
 require "dry/validation"
 
-# standard & utility libs
+# Grape
+require "grape"
+require "grape-entity"
+
+# Other utility libs
 require "addressable/uri"
 require "awrence"
-require "colorize"
 require "email_address"
-require "ipaddr"
-require "json"
 require "memist"
-require "net/http"
-require "net/https"
 require "net/ping"
+require "parallel"
 require "plissken"
 require "public_suffix"
-require "resolv"
+require "semantic_logger"
+require "sentry-ruby"
 require "uuidtools"
-require "yaml"
-require "parallel"
 
 # Load .env
 require "dotenv/load"
@@ -47,6 +56,7 @@ require "mihari/mixins/autonomous_system"
 require "mihari/mixins/configurable"
 require "mihari/mixins/database"
 require "mihari/mixins/disallowed_data_value"
+require "mihari/mixins/error_notification"
 require "mihari/mixins/refang"
 require "mihari/mixins/retriable"
 require "mihari/mixins/rule"
@@ -88,6 +98,7 @@ module Mihari
   setting :webhook_url, default: ENV["WEBHOOK_URL"]
   setting :webhook_use_json_body, constructor: ->(value = ENV["WEBHOOK_USE_JSON_BODY"]) { truthy?(value) }
   setting :zoomeye_api_key, default: ENV["ZOOMEYE_API_KEY"]
+  setting :sentry_dsn, default: ENV["SENTRY_DSN"]
 
   class << self
     include Memist::Memoizable
@@ -106,12 +117,32 @@ module Mihari
       []
     end
     memoize :enrichers
+
+    def logger
+      SemanticLogger.default_level = :info
+      SemanticLogger.add_appender(io: $stderr, formatter: :color)
+      SemanticLogger["Mihari"]
+    end
+    memoize :logger
+
+    def initialize_sentry
+      return if Mihari.config.sentry_dsn.nil?
+      return if Sentry.initialized?
+
+      Sentry.init do |config|
+        config.dsn = Mihari.config.sentry_dsn
+
+        config.traces_sample_rate = 0.5
+      end
+    end
   end
 end
 
 require "mihari/database"
 require "mihari/type_checker"
 
+require "mihari/http"
+
 # Constants
 require "mihari/constants"
 
@@ -151,6 +182,16 @@ require "mihari/models/tag"
 require "mihari/models/tagging"
 require "mihari/models/whois"
 
+# Emitters
+require "mihari/emitters/base"
+
+require "mihari/emitters/database"
+require "mihari/emitters/http"
+require "mihari/emitters/misp"
+require "mihari/emitters/slack"
+require "mihari/emitters/the_hive"
+require "mihari/emitters/webhook"
+
 # Analyzers
 require "mihari/analyzers/base"
 
@@ -175,19 +216,25 @@ require "mihari/analyzers/virustotal"
 require "mihari/analyzers/zoomeye"
 require "mihari/analyzers/rule"
 
-# Notifiers
-require "mihari/notifiers/base"
-require "mihari/notifiers/slack"
-require "mihari/notifiers/exception_notifier"
+# Entities
+require "mihari/entities/message"
 
-# Emitters
-require "mihari/emitters/base"
-require "mihari/emitters/database"
-require "mihari/emitters/misp"
-require "mihari/emitters/slack"
-require "mihari/emitters/stdout"
-require "mihari/emitters/the_hive"
-require "mihari/emitters/webhook"
+require "mihari/entities/autonomous_system"
+require "mihari/entities/command"
+require "mihari/entities/config"
+require "mihari/entities/dns"
+require "mihari/entities/geolocation"
+require "mihari/entities/ip_address"
+require "mihari/entities/reverse_dns"
+require "mihari/entities/source"
+require "mihari/entities/tag"
+require "mihari/entities/whois"
+
+require "mihari/entities/artifact"
+
+require "mihari/entities/alert"
+
+require "mihari/entities/rule"
 
 # Status checker
 require "mihari/status"
@@ -197,3 +244,6 @@ require "mihari/web/app"
 
 # CLIs
 require "mihari/cli/main"
+
+# initialize Sentry
+Mihari.initialize_sentry

data/mihari.gemspec

@@ -16,6 +16,8 @@ Gem::Specification.new do |spec|
 
   spec.required_ruby_version = ">= 2.7"
 
+  spec.metadata["rubygems_mfa_required"] = "true"
+
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
   spec.files = Dir.chdir(File.expand_path(__dir__)) do
@@ -26,7 +28,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_development_dependency "bundler", "~> 2.3"
-  spec.add_development_dependency "coveralls_reborn", "~> 0.23"
+  spec.add_development_dependency "coveralls_reborn", "~> 0.24"
   spec.add_development_dependency "fakefs", "~> 1.4"
   spec.add_development_dependency "mysql2", "~> 0.5"
   spec.add_development_dependency "overcommit", "~> 0.58"
@@ -35,19 +37,19 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rb-fsevent", "~> 0.11"
   spec.add_development_dependency "rerun", "~> 0.13"
-  spec.add_development_dependency "rspec", "~> 3.10"
-  spec.add_development_dependency "standard", "~> 1.7"
-  spec.add_development_dependency "steep", "~> 0.47"
+  spec.add_development_dependency "rspec", "~> 3.11"
+  spec.add_development_dependency "simplecov-lcov", "~> 0.8.0"
+  spec.add_development_dependency "standard", "~> 1.8"
+  spec.add_development_dependency "steep", "~> 0.49"
   spec.add_development_dependency "timecop", "~> 0.9"
-  spec.add_development_dependency "vcr", "~> 6.0"
+  spec.add_development_dependency "vcr", "~> 6.1"
   spec.add_development_dependency "webmock", "~> 3.14"
 
-  spec.add_dependency "activerecord", "7.0.1"
+  spec.add_dependency "activerecord", "7.0.2.3"
   spec.add_dependency "addressable", "~> 2.8"
   spec.add_dependency "awrence", "~> 2.0"
   spec.add_dependency "binaryedge", "~> 0.1"
   spec.add_dependency "censysx", "~> 0.1"
-  spec.add_dependency "colorize", "~> 0.8"
   spec.add_dependency "crtsh-rb", "~> 0.3"
   spec.add_dependency "dnpedia", "~> 0.1"
   spec.add_dependency "dnstwister", "~> 0.1"
@@ -56,8 +58,9 @@ Gem::Specification.new do |spec|
   spec.add_dependency "dry-container", "0.9.0"
   spec.add_dependency "dry-files", "0.1.0"
   spec.add_dependency "dry-initializer", "3.1.1"
+  spec.add_dependency "dry-schema", "1.9.1"
   spec.add_dependency "dry-struct", "1.4.0"
-  spec.add_dependency "dry-validation", "1.7.0"
+  spec.add_dependency "dry-validation", "1.8.0"
   spec.add_dependency "email_address", "~> 0.2"
   spec.add_dependency "grape", "1.6.2"
   spec.add_dependency "grape-entity", "0.10.1"
@@ -65,7 +68,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "grape-swagger-entity", "0.5.1"
   spec.add_dependency "greynoise", "~> 0.1"
   spec.add_dependency "hachi", "~> 1.0"
-  spec.add_dependency "http", "~> 5.0"
+  spec.add_dependency "insensitive_hash", "~> 0.3"
   spec.add_dependency "jr-cli", "~> 0.5"
   spec.add_dependency "launchy", "~> 2.5"
   spec.add_dependency "memist", "~> 2.0"
@@ -80,17 +83,19 @@ Gem::Specification.new do |spec|
   spec.add_dependency "plissken", "~> 2.0"
   spec.add_dependency "public_suffix", "~> 4.0"
   spec.add_dependency "pulsedive", "~> 0.1"
-  spec.add_dependency "puma", "5.6.1"
+  spec.add_dependency "puma", "5.6.2"
   spec.add_dependency "rack", "2.2.3"
   spec.add_dependency "rack-contrib", "2.3.0"
   spec.add_dependency "rack-cors", "~> 1.1"
   spec.add_dependency "safe_shell", "~> 1.1"
   spec.add_dependency "securitytrails", "~> 1.0"
+  spec.add_dependency "semantic_logger", "~> 4.10"
+  spec.add_dependency "sentry-ruby", "~> 5.1.1"
   spec.add_dependency "shodanx", "~> 0.2"
   spec.add_dependency "slack-notifier", "~> 2.4"
   spec.add_dependency "spysex", "~> 0.2"
   spec.add_dependency "sqlite3", "~> 1.4"
-  spec.add_dependency "thor", "1.1.0"
+  spec.add_dependency "thor", "1.2.1"
   spec.add_dependency "thread_safe", "~> 0.3"
   spec.add_dependency "urlscan", "~> 0.8"
   spec.add_dependency "uuidtools", "~> 2.2"

data/sig/lib/mihari/constants.rbs

@@ -1,3 +1,5 @@
 module Mihari
   ALLOWED_DATA_TYPES: Array[String]
+
+  DEFAULT_EMITTERS: Array[Hash]
 end

data/sig/lib/mihari/emitters/http.rbs

@@ -0,0 +1,35 @@
+module Mihari
+  module Emitters
+    class PayloadTemplate < ERB
+      def self.template: () -> ::String
+
+      def initialize: (title: untyped title, description: untyped description, artifacts: untyped artifacts, source: untyped source, tags: untyped tags, ?options: ::Hash[untyped, untyped] options) -> void
+
+      def result: () -> untyped
+    end
+
+    class HTTP < Base
+      # @return [Addressable::URI, nil]
+      attr_reader uri: Addressable::URI?
+
+      # @return [Hash]
+      attr_reader http_request_headers: Hash
+
+      # @return [String]
+      attr_reader http_request_method: String
+
+      # @return [String, nil]
+      attr_reader template: String?
+
+      def initialize: (*untyped args, **untyped kwargs) -> void
+
+      def emit: (title: String title, artifacts: Array[Mihari::Artifact] artifacts, ?tags: Array[String] tags, **untyped _options) -> void
+
+      def valid?: () -> bool
+
+      private
+
+      def payload_as_string: (title: String title, description: String description, artifacts: Array[Mihari::Artifact] artifacts, source: String source, tags: Array[String] tags) -> String
+    end
+  end
+end

data/sig/lib/mihari/emitters/slack.rbs

@@ -39,7 +39,35 @@ module Mihari
     end
 
     class Slack < Base
-      def notifier: () -> Mihari::Notifiers::Slack
+      SLACK_WEBHOOK_URL_KEY: ::String
+
+      SLACK_CHANNEL_KEY: ::String
+
+      DEFAULT_USERNAME: ::String
+
+      #
+      # Slack channel to post
+      #
+      # @return [String]
+      #
+      def slack_channel: () -> String
+
+      #
+      # Slack webhook URL
+      #
+      # @return [String]
+      #
+      def slack_webhook_url: () -> String
+
+      #
+      # Check Slack webhook URL is set
+      #
+      # @return [Boolean]
+      #
+      def slack_webhook_url?: () -> bool
+
+
+      def notifier: () -> ::Slack::Notifier
 
       def valid?: () -> bool
 

data/sig/lib/mihari/feed/reader.rbs

@@ -12,8 +12,8 @@ module Mihari
 
     def initialize: (
       String uri,
-      ?http_request_headers: Hash[(String | Symbol), untyped]] http_request_headers,
-      ?http_request_method: ::String http_request_method,
+      ?http_request_headers: Hash[(String | Symbol), untyped] http_request_headers,
+      ?http_request_method: String http_request_method,
       ?http_request_payload_type: String?  http_request_payload_type,
       ?http_request_payload: Hash[(String | Symbol), untyped] http_request_payload
     ) -> void

data/sig/lib/mihari/http.rbs

@@ -0,0 +1,64 @@
+
+module Mihari
+  class HTTP
+    attr_reader uri: URI
+
+    attr_reader headers: Hash[(String | Symbol), untyped]
+
+    attr_reader payload: Hash[(String | Symbol), untyped]
+
+    def initialize: (
+      String uri,
+      ?headers: Hash[(String | Symbol), untyped] headers,
+      ?payload: Hash[(String | Symbol), untyped] payload
+    ) -> void
+
+    #
+    # Make a GET request
+    #
+    # @return [Net::HTTPResponse]
+    #
+    def get: () -> Net::HTTPResponse
+
+    #
+    # Make a POST request
+    #
+    # @return [Net::HTTPResponse]
+    #
+    def post: () -> Net::HTTPResponse
+
+    def self.get: (
+      String uri,
+      ?headers: Hash[(String | Symbol), untyped] headers,
+      ?payload_type: String? payload_type,
+      ?payload: Hash[(String | Symbol), untyped] payload
+    ) -> Net::HTTPResponse
+
+    def self.post: (
+      String uri,
+      ?headers: Hash[(String | Symbol), untyped] headers,
+      ?payload_type: String? payload_type,
+      ?payload: Hash[(String | Symbol), untyped] payload
+    ) -> Net::HTTPResponse
+
+    private
+
+    def content_type: () -> String
+
+    #
+    # Get options for HTTP request
+    #
+    # @return [Hahs]
+    #
+    def https_options: () -> ({ use_ssl: ::TrueClass } | ::Hash[untyped, untyped])
+
+    #
+    # Make a HTTP request
+    #
+    # @param [Net::HTTPRequest] req
+    #
+    # @return [Net::HTTPResponse]
+    #
+    def request: (untyped req) -> Net::HTTPResponse
+  end
+end

data/sig/lib/mihari/mixins/error_notification.rbs

@@ -0,0 +1,12 @@
+module Mihari
+  module Mixins
+    module ErrorNotification
+			#
+			# Send an exception notification if there is any error in a block
+			#
+			# @return [Nil]
+			#
+			def with_error_notification: () { () -> untyped } -> void
+    end
+  end
+end

data/sig/lib/mihari/structs/rule.rbs

@@ -50,6 +50,10 @@ module Mihari
         # @return [Mihari::Rule]
         #
         def to_model: () -> Mihari::Rule
+
+        def to_analyzer: () -> Mihari::Analyzers::Rule
+
+        def self.from_model: (Mihari::Rule model) -> Mihari::Structs::Rule::Rule
       end
     end
   end

data/sig/lib/mihari/types.rbs

@@ -17,5 +17,7 @@ module Mihari
     FeedHttpRequestMethods: Array[String]
 
     FeedHttpRequestPayloadTypes: Array[String]
+
+    EmitterTypes: Array[String]
   end
 end

data/sig/lib/mihari.rbs

@@ -25,6 +25,7 @@ class Configuration
   attr_accessor webhook_url (): String?
   attr_accessor webhook_use_json_body (): (bool | nil)
   attr_accessor database (): String?
+  attr_accessor sentry_dsn(): String?
 
   attr_reader values: Hash[(String | Symbol), String?]
 end
@@ -42,14 +43,9 @@ module Mihari
 
   def self.enrichers: () -> ::Array[singleton(Mihari::Enrichers::Base)]
 
-  #
-  # Load configuration from YAML file
-  #
-  # @param [String] path Path to YAML file
-  #
-  # @return [nil]
-  #
-  def self.load_config_from_yaml: (String path) -> void
+  def self.logger: () -> SemanticLogger
+
+  def self.initialize_sentry: () -> void
 end
 
 class Object