mihari 4.1.0 → 4.2.0

data/lib/mihari.rb

@@ -1,5 +1,14 @@
 # frozen_string_literal: true
 
+# standard libs
+require "ipaddr"
+require "json"
+require "net/http"
+require "net/https"
+require "resolv"
+require "yaml"
+
+# Active Support & Active Record
 require "active_support"
 
 require "active_support/core_ext/hash"
@@ -17,23 +26,22 @@ require "dry/struct"
 require "dry/types"
 require "dry/validation"
 
-# standard & utility libs
+# Grape
+require "grape"
+require "grape-entity"
+
+# Other utility libs
 require "addressable/uri"
 require "awrence"
-require "colorize"
 require "email_address"
-require "ipaddr"
-require "json"
 require "memist"
-require "net/http"
-require "net/https"
 require "net/ping"
+require "parallel"
 require "plissken"
 require "public_suffix"
-require "resolv"
+require "semantic_logger"
+require "sentry-ruby"
 require "uuidtools"
-require "yaml"
-require "parallel"
 
 # Load .env
 require "dotenv/load"
@@ -47,6 +55,7 @@ require "mihari/mixins/autonomous_system"
 require "mihari/mixins/configurable"
 require "mihari/mixins/database"
 require "mihari/mixins/disallowed_data_value"
+require "mihari/mixins/error_notification"
 require "mihari/mixins/refang"
 require "mihari/mixins/retriable"
 require "mihari/mixins/rule"
@@ -88,6 +97,7 @@ module Mihari
   setting :webhook_url, default: ENV["WEBHOOK_URL"]
   setting :webhook_use_json_body, constructor: ->(value = ENV["WEBHOOK_USE_JSON_BODY"]) { truthy?(value) }
   setting :zoomeye_api_key, default: ENV["ZOOMEYE_API_KEY"]
+  setting :sentry_dsn, default: ENV["SENTRY_DSN"]
 
   class << self
     include Memist::Memoizable
@@ -106,12 +116,32 @@ module Mihari
       []
     end
     memoize :enrichers
+
+    def logger
+      SemanticLogger.default_level = :info
+      SemanticLogger.add_appender(io: $stderr, formatter: :color)
+      SemanticLogger["Mihari"]
+    end
+    memoize :logger
+
+    def initialize_sentry
+      return if Mihari.config.sentry_dsn.nil?
+      return if Sentry.initialized?
+
+      Sentry.init do |config|
+        config.dsn = Mihari.config.sentry_dsn
+
+        config.traces_sample_rate = 0.5
+      end
+    end
   end
 end
 
 require "mihari/database"
 require "mihari/type_checker"
 
+require "mihari/http"
+
 # Constants
 require "mihari/constants"
 
@@ -175,20 +205,35 @@ require "mihari/analyzers/virustotal"
 require "mihari/analyzers/zoomeye"
 require "mihari/analyzers/rule"
 
-# Notifiers
-require "mihari/notifiers/base"
-require "mihari/notifiers/slack"
-require "mihari/notifiers/exception_notifier"
-
 # Emitters
 require "mihari/emitters/base"
 require "mihari/emitters/database"
 require "mihari/emitters/misp"
 require "mihari/emitters/slack"
-require "mihari/emitters/stdout"
 require "mihari/emitters/the_hive"
 require "mihari/emitters/webhook"
 
+# Entities
+
+require "mihari/entities/message"
+
+require "mihari/entities/autonomous_system"
+require "mihari/entities/command"
+require "mihari/entities/config"
+require "mihari/entities/dns"
+require "mihari/entities/geolocation"
+require "mihari/entities/ip_address"
+require "mihari/entities/reverse_dns"
+require "mihari/entities/source"
+require "mihari/entities/tag"
+require "mihari/entities/whois"
+
+require "mihari/entities/artifact"
+
+require "mihari/entities/alert"
+
+require "mihari/entities/rule"
+
 # Status checker
 require "mihari/status"
 
@@ -197,3 +242,6 @@ require "mihari/web/app"
 
 # CLIs
 require "mihari/cli/main"
+
+# initialize Sentry
+Mihari.initialize_sentry

data/mihari.gemspec

@@ -30,24 +30,23 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "fakefs", "~> 1.4"
   spec.add_development_dependency "mysql2", "~> 0.5"
   spec.add_development_dependency "overcommit", "~> 0.58"
-  spec.add_development_dependency "pg", "~> 1.2"
+  spec.add_development_dependency "pg", "~> 1.3"
   spec.add_development_dependency "rack-test", "~> 1.1"
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rb-fsevent", "~> 0.11"
   spec.add_development_dependency "rerun", "~> 0.13"
-  spec.add_development_dependency "rspec", "~> 3.10"
-  spec.add_development_dependency "standard", "~> 1.5"
+  spec.add_development_dependency "rspec", "~> 3.11"
+  spec.add_development_dependency "standard", "~> 1.7"
   spec.add_development_dependency "steep", "~> 0.47"
   spec.add_development_dependency "timecop", "~> 0.9"
   spec.add_development_dependency "vcr", "~> 6.0"
   spec.add_development_dependency "webmock", "~> 3.14"
 
-  spec.add_dependency "activerecord", "7.0.1"
+  spec.add_dependency "activerecord", "7.0.2.2"
   spec.add_dependency "addressable", "~> 2.8"
-  spec.add_dependency "awrence", "~> 1.2"
+  spec.add_dependency "awrence", "~> 2.0"
   spec.add_dependency "binaryedge", "~> 0.1"
   spec.add_dependency "censysx", "~> 0.1"
-  spec.add_dependency "colorize", "~> 0.8"
   spec.add_dependency "crtsh-rb", "~> 0.3"
   spec.add_dependency "dnpedia", "~> 0.1"
   spec.add_dependency "dnstwister", "~> 0.1"
@@ -57,7 +56,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "dry-files", "0.1.0"
   spec.add_dependency "dry-initializer", "3.1.1"
   spec.add_dependency "dry-struct", "1.4.0"
-  spec.add_dependency "dry-validation", "1.7.0"
+  spec.add_dependency "dry-validation", "1.8.0"
   spec.add_dependency "email_address", "~> 0.2"
   spec.add_dependency "grape", "1.6.2"
   spec.add_dependency "grape-entity", "0.10.1"
@@ -65,32 +64,33 @@ Gem::Specification.new do |spec|
   spec.add_dependency "grape-swagger-entity", "0.5.1"
   spec.add_dependency "greynoise", "~> 0.1"
   spec.add_dependency "hachi", "~> 1.0"
-  spec.add_dependency "http", "~> 5.0"
   spec.add_dependency "jr-cli", "~> 0.5"
   spec.add_dependency "launchy", "~> 2.5"
   spec.add_dependency "memist", "~> 2.0"
   spec.add_dependency "misp", "~> 0.1"
   spec.add_dependency "net-ping", "~> 2.0"
-  spec.add_dependency "normalize_country", "0.3"
+  spec.add_dependency "normalize_country", "~> 0.3"
   spec.add_dependency "onyphe", "~> 2.0"
   spec.add_dependency "otx_ruby", "~> 0.9"
   spec.add_dependency "parallel", "~> 1.21"
   spec.add_dependency "passive_circl", "~> 0.1"
   spec.add_dependency "passivetotalx", "~> 0.1"
-  spec.add_dependency "plissken", "~> 1.4"
+  spec.add_dependency "plissken", "~> 2.0"
   spec.add_dependency "public_suffix", "~> 4.0"
   spec.add_dependency "pulsedive", "~> 0.1"
-  spec.add_dependency "puma", "5.5.2"
+  spec.add_dependency "puma", "5.6.2"
   spec.add_dependency "rack", "2.2.3"
   spec.add_dependency "rack-contrib", "2.3.0"
   spec.add_dependency "rack-cors", "~> 1.1"
   spec.add_dependency "safe_shell", "~> 1.1"
   spec.add_dependency "securitytrails", "~> 1.0"
+  spec.add_dependency "semantic_logger", "~> 4.10"
+  spec.add_dependency "sentry-ruby", "~> 5.1.1"
   spec.add_dependency "shodanx", "~> 0.2"
   spec.add_dependency "slack-notifier", "~> 2.4"
   spec.add_dependency "spysex", "~> 0.2"
   spec.add_dependency "sqlite3", "~> 1.4"
-  spec.add_dependency "thor", "1.1.0"
+  spec.add_dependency "thor", "1.2.1"
   spec.add_dependency "thread_safe", "~> 0.3"
   spec.add_dependency "urlscan", "~> 0.8"
   spec.add_dependency "uuidtools", "~> 2.2"

data/sig/lib/mihari/emitters/slack.rbs

@@ -39,7 +39,35 @@ module Mihari
     end
 
     class Slack < Base
-      def notifier: () -> Mihari::Notifiers::Slack
+      SLACK_WEBHOOK_URL_KEY: ::String
+
+      SLACK_CHANNEL_KEY: ::String
+
+      DEFAULT_USERNAME: ::String
+
+      #
+      # Slack channel to post
+      #
+      # @return [String]
+      #
+      def slack_channel: () -> String
+
+      #
+      # Slack webhook URL
+      #
+      # @return [String]
+      #
+      def slack_webhook_url: () -> String
+
+      #
+      # Check Slack webhook URL is set
+      #
+      # @return [Boolean]
+      #
+      def slack_webhook_url?: () -> bool
+
+
+      def notifier: () -> ::Slack::Notifier
 
       def valid?: () -> bool
 

data/sig/lib/mihari/feed/reader.rbs

@@ -12,8 +12,8 @@ module Mihari
 
     def initialize: (
       String uri,
-      ?http_request_headers: Hash[(String | Symbol), untyped]] http_request_headers,
-      ?http_request_method: ::String http_request_method,
+      ?http_request_headers: Hash[(String | Symbol), untyped] http_request_headers,
+      ?http_request_method: String http_request_method,
       ?http_request_payload_type: String?  http_request_payload_type,
       ?http_request_payload: Hash[(String | Symbol), untyped] http_request_payload
     ) -> void

data/sig/lib/mihari/http.rbs

@@ -0,0 +1,65 @@
+
+module Mihari
+  class HTTP
+    attr_reader uri: URI
+
+    attr_reader headers: Hash[(String | Symbol), untyped]
+
+    attr_reader payload_type: String?
+
+    attr_reader payload: Hash[(String | Symbol), untyped]
+
+    def initialize: (
+      String uri,
+      ?headers: Hash[(String | Symbol), untyped] headers,
+      ?payload_type: String? payload_type,
+      ?payload: Hash[(String | Symbol), untyped] payload
+    ) -> void
+
+    #
+    # Make a GET request
+    #
+    # @return [Net::HTTPResponse]
+    #
+    def get: () -> Net::HTTPResponse
+
+    #
+    # Make a POST request
+    #
+    # @return [Net::HTTPResponse]
+    #
+    def post: () -> Net::HTTPResponse
+
+    def self.get: (
+      String uri,
+      ?headers: Hash[(String | Symbol), untyped] headers,
+      ?payload_type: String? payload_type,
+      ?payload: Hash[(String | Symbol), untyped] payload
+    ) -> Net::HTTPResponse
+
+    def self.post: (
+      String uri,
+      ?headers: Hash[(String | Symbol), untyped] headers,
+      ?payload_type: String? payload_type,
+      ?payload: Hash[(String | Symbol), untyped] payload
+    ) -> Net::HTTPResponse
+
+    private
+
+    #
+    # Get options for HTTP request
+    #
+    # @return [Hahs]
+    #
+    def https_options: () -> ({ use_ssl: ::TrueClass } | ::Hash[untyped, untyped])
+
+    #
+    # Make a HTTP request
+    #
+    # @param [Net::HTTPRequest] req
+    #
+    # @return [Net::HTTPResponse]
+    #
+    def request: (untyped req) -> Net::HTTPResponse
+  end
+end

data/sig/lib/mihari/mixins/error_notification.rbs

@@ -0,0 +1,12 @@
+module Mihari
+  module Mixins
+    module ErrorNotification
+			#
+			# Send an exception notification if there is any error in a block
+			#
+			# @return [Nil]
+			#
+			def with_error_notification: () { () -> untyped } -> void
+    end
+  end
+end

data/sig/lib/mihari/structs/rule.rbs

@@ -50,6 +50,12 @@ module Mihari
         # @return [Mihari::Rule]
         #
         def to_model: () -> Mihari::Rule
+
+        def to_analyzer: () -> Mihari::Analyzers::Rule
+
+        class << self
+          def from_model: (Mihari::Rule model) -> Mihari::Structs::Rule::Rule
+        end
       end
     end
   end

data/sig/lib/mihari.rbs

@@ -25,6 +25,7 @@ class Configuration
   attr_accessor webhook_url (): String?
   attr_accessor webhook_use_json_body (): (bool | nil)
   attr_accessor database (): String?
+  attr_accessor sentry_dsn(): String?
 
   attr_reader values: Hash[(String | Symbol), String?]
 end
@@ -42,14 +43,9 @@ module Mihari
 
   def self.enrichers: () -> ::Array[singleton(Mihari::Enrichers::Base)]
 
-  #
-  # Load configuration from YAML file
-  #
-  # @param [String] path Path to YAML file
-  #
-  # @return [nil]
-  #
-  def self.load_config_from_yaml: (String path) -> void
+  def self.logger: () -> SemanticLogger
+
+  def self.initialize_sentry: () -> void
 end
 
 class Object