mihari 4.1.2 → 4.2.0

data/lib/mihari.rb

@@ -1,5 +1,14 @@
 # frozen_string_literal: true
 
+# standard libs
+require "ipaddr"
+require "json"
+require "net/http"
+require "net/https"
+require "resolv"
+require "yaml"
+
+# Active Support & Active Record
 require "active_support"
 
 require "active_support/core_ext/hash"
@@ -17,23 +26,22 @@ require "dry/struct"
 require "dry/types"
 require "dry/validation"
 
-# standard & utility libs
+# Grape
+require "grape"
+require "grape-entity"
+
+# Other utility libs
 require "addressable/uri"
 require "awrence"
-require "colorize"
 require "email_address"
-require "ipaddr"
-require "json"
 require "memist"
-require "net/http"
-require "net/https"
 require "net/ping"
+require "parallel"
 require "plissken"
 require "public_suffix"
-require "resolv"
+require "semantic_logger"
+require "sentry-ruby"
 require "uuidtools"
-require "yaml"
-require "parallel"
 
 # Load .env
 require "dotenv/load"
@@ -47,6 +55,7 @@ require "mihari/mixins/autonomous_system"
 require "mihari/mixins/configurable"
 require "mihari/mixins/database"
 require "mihari/mixins/disallowed_data_value"
+require "mihari/mixins/error_notification"
 require "mihari/mixins/refang"
 require "mihari/mixins/retriable"
 require "mihari/mixins/rule"
@@ -88,6 +97,7 @@ module Mihari
   setting :webhook_url, default: ENV["WEBHOOK_URL"]
   setting :webhook_use_json_body, constructor: ->(value = ENV["WEBHOOK_USE_JSON_BODY"]) { truthy?(value) }
   setting :zoomeye_api_key, default: ENV["ZOOMEYE_API_KEY"]
+  setting :sentry_dsn, default: ENV["SENTRY_DSN"]
 
   class << self
     include Memist::Memoizable
@@ -106,12 +116,32 @@ module Mihari
       []
     end
     memoize :enrichers
+
+    def logger
+      SemanticLogger.default_level = :info
+      SemanticLogger.add_appender(io: $stderr, formatter: :color)
+      SemanticLogger["Mihari"]
+    end
+    memoize :logger
+
+    def initialize_sentry
+      return if Mihari.config.sentry_dsn.nil?
+      return if Sentry.initialized?
+
+      Sentry.init do |config|
+        config.dsn = Mihari.config.sentry_dsn
+
+        config.traces_sample_rate = 0.5
+      end
+    end
   end
 end
 
 require "mihari/database"
 require "mihari/type_checker"
 
+require "mihari/http"
+
 # Constants
 require "mihari/constants"
 
@@ -175,20 +205,35 @@ require "mihari/analyzers/virustotal"
 require "mihari/analyzers/zoomeye"
 require "mihari/analyzers/rule"
 
-# Notifiers
-require "mihari/notifiers/base"
-require "mihari/notifiers/slack"
-require "mihari/notifiers/exception_notifier"
-
 # Emitters
 require "mihari/emitters/base"
 require "mihari/emitters/database"
 require "mihari/emitters/misp"
 require "mihari/emitters/slack"
-require "mihari/emitters/stdout"
 require "mihari/emitters/the_hive"
 require "mihari/emitters/webhook"
 
+# Entities
+
+require "mihari/entities/message"
+
+require "mihari/entities/autonomous_system"
+require "mihari/entities/command"
+require "mihari/entities/config"
+require "mihari/entities/dns"
+require "mihari/entities/geolocation"
+require "mihari/entities/ip_address"
+require "mihari/entities/reverse_dns"
+require "mihari/entities/source"
+require "mihari/entities/tag"
+require "mihari/entities/whois"
+
+require "mihari/entities/artifact"
+
+require "mihari/entities/alert"
+
+require "mihari/entities/rule"
+
 # Status checker
 require "mihari/status"
 
@@ -197,3 +242,6 @@ require "mihari/web/app"
 
 # CLIs
 require "mihari/cli/main"
+
+# initialize Sentry
+Mihari.initialize_sentry

data/mihari.gemspec

@@ -47,7 +47,6 @@ Gem::Specification.new do |spec|
   spec.add_dependency "awrence", "~> 2.0"
   spec.add_dependency "binaryedge", "~> 0.1"
   spec.add_dependency "censysx", "~> 0.1"
-  spec.add_dependency "colorize", "~> 0.8"
   spec.add_dependency "crtsh-rb", "~> 0.3"
   spec.add_dependency "dnpedia", "~> 0.1"
   spec.add_dependency "dnstwister", "~> 0.1"
@@ -57,7 +56,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "dry-files", "0.1.0"
   spec.add_dependency "dry-initializer", "3.1.1"
   spec.add_dependency "dry-struct", "1.4.0"
-  spec.add_dependency "dry-validation", "1.7.0"
+  spec.add_dependency "dry-validation", "1.8.0"
   spec.add_dependency "email_address", "~> 0.2"
   spec.add_dependency "grape", "1.6.2"
   spec.add_dependency "grape-entity", "0.10.1"
@@ -65,7 +64,6 @@ Gem::Specification.new do |spec|
   spec.add_dependency "grape-swagger-entity", "0.5.1"
   spec.add_dependency "greynoise", "~> 0.1"
   spec.add_dependency "hachi", "~> 1.0"
-  spec.add_dependency "http", "~> 5.0"
   spec.add_dependency "jr-cli", "~> 0.5"
   spec.add_dependency "launchy", "~> 2.5"
   spec.add_dependency "memist", "~> 2.0"
@@ -86,6 +84,8 @@ Gem::Specification.new do |spec|
   spec.add_dependency "rack-cors", "~> 1.1"
   spec.add_dependency "safe_shell", "~> 1.1"
   spec.add_dependency "securitytrails", "~> 1.0"
+  spec.add_dependency "semantic_logger", "~> 4.10"
+  spec.add_dependency "sentry-ruby", "~> 5.1.1"
   spec.add_dependency "shodanx", "~> 0.2"
   spec.add_dependency "slack-notifier", "~> 2.4"
   spec.add_dependency "spysex", "~> 0.2"

data/sig/lib/mihari/emitters/slack.rbs

@@ -39,7 +39,35 @@ module Mihari
     end
 
     class Slack < Base
-      def notifier: () -> Mihari::Notifiers::Slack
+      SLACK_WEBHOOK_URL_KEY: ::String
+
+      SLACK_CHANNEL_KEY: ::String
+
+      DEFAULT_USERNAME: ::String
+
+      #
+      # Slack channel to post
+      #
+      # @return [String]
+      #
+      def slack_channel: () -> String
+
+      #
+      # Slack webhook URL
+      #
+      # @return [String]
+      #
+      def slack_webhook_url: () -> String
+
+      #
+      # Check Slack webhook URL is set
+      #
+      # @return [Boolean]
+      #
+      def slack_webhook_url?: () -> bool
+
+
+      def notifier: () -> ::Slack::Notifier
 
       def valid?: () -> bool
 

data/sig/lib/mihari/feed/reader.rbs

@@ -12,8 +12,8 @@ module Mihari
 
     def initialize: (
       String uri,
-      ?http_request_headers: Hash[(String | Symbol), untyped]] http_request_headers,
-      ?http_request_method: ::String http_request_method,
+      ?http_request_headers: Hash[(String | Symbol), untyped] http_request_headers,
+      ?http_request_method: String http_request_method,
       ?http_request_payload_type: String?  http_request_payload_type,
       ?http_request_payload: Hash[(String | Symbol), untyped] http_request_payload
     ) -> void

data/sig/lib/mihari/http.rbs

@@ -0,0 +1,65 @@
+
+module Mihari
+  class HTTP
+    attr_reader uri: URI
+
+    attr_reader headers: Hash[(String | Symbol), untyped]
+
+    attr_reader payload_type: String?
+
+    attr_reader payload: Hash[(String | Symbol), untyped]
+
+    def initialize: (
+      String uri,
+      ?headers: Hash[(String | Symbol), untyped] headers,
+      ?payload_type: String? payload_type,
+      ?payload: Hash[(String | Symbol), untyped] payload
+    ) -> void
+
+    #
+    # Make a GET request
+    #
+    # @return [Net::HTTPResponse]
+    #
+    def get: () -> Net::HTTPResponse
+
+    #
+    # Make a POST request
+    #
+    # @return [Net::HTTPResponse]
+    #
+    def post: () -> Net::HTTPResponse
+
+    def self.get: (
+      String uri,
+      ?headers: Hash[(String | Symbol), untyped] headers,
+      ?payload_type: String? payload_type,
+      ?payload: Hash[(String | Symbol), untyped] payload
+    ) -> Net::HTTPResponse
+
+    def self.post: (
+      String uri,
+      ?headers: Hash[(String | Symbol), untyped] headers,
+      ?payload_type: String? payload_type,
+      ?payload: Hash[(String | Symbol), untyped] payload
+    ) -> Net::HTTPResponse
+
+    private
+
+    #
+    # Get options for HTTP request
+    #
+    # @return [Hahs]
+    #
+    def https_options: () -> ({ use_ssl: ::TrueClass } | ::Hash[untyped, untyped])
+
+    #
+    # Make a HTTP request
+    #
+    # @param [Net::HTTPRequest] req
+    #
+    # @return [Net::HTTPResponse]
+    #
+    def request: (untyped req) -> Net::HTTPResponse
+  end
+end

data/sig/lib/mihari/mixins/error_notification.rbs

@@ -0,0 +1,12 @@
+module Mihari
+  module Mixins
+    module ErrorNotification
+			#
+			# Send an exception notification if there is any error in a block
+			#
+			# @return [Nil]
+			#
+			def with_error_notification: () { () -> untyped } -> void
+    end
+  end
+end

data/sig/lib/mihari/structs/rule.rbs

@@ -50,6 +50,12 @@ module Mihari
         # @return [Mihari::Rule]
         #
         def to_model: () -> Mihari::Rule
+
+        def to_analyzer: () -> Mihari::Analyzers::Rule
+
+        class << self
+          def from_model: (Mihari::Rule model) -> Mihari::Structs::Rule::Rule
+        end
       end
     end
   end

data/sig/lib/mihari.rbs

@@ -25,6 +25,7 @@ class Configuration
   attr_accessor webhook_url (): String?
   attr_accessor webhook_use_json_body (): (bool | nil)
   attr_accessor database (): String?
+  attr_accessor sentry_dsn(): String?
 
   attr_reader values: Hash[(String | Symbol), String?]
 end
@@ -42,14 +43,9 @@ module Mihari
 
   def self.enrichers: () -> ::Array[singleton(Mihari::Enrichers::Base)]
 
-  #
-  # Load configuration from YAML file
-  #
-  # @param [String] path Path to YAML file
-  #
-  # @return [nil]
-  #
-  def self.load_config_from_yaml: (String path) -> void
+  def self.logger: () -> SemanticLogger
+
+  def self.initialize_sentry: () -> void
 end
 
 class Object

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mihari
 version: !ruby/object:Gem::Version
-  version: 4.1.2
+  version: 4.2.0
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-02-12 00:00:00.000000000 Z
+date: 2022-03-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -304,20 +304,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.1'
-- !ruby/object:Gem::Dependency
-  name: colorize
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.8'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.8'
 - !ruby/object:Gem::Dependency
   name: crtsh-rb
   requirement: !ruby/object:Gem::Requirement
@@ -450,14 +436,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.7.0
+        version: 1.8.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.7.0
+        version: 1.8.0
 - !ruby/object:Gem::Dependency
   name: email_address
   requirement: !ruby/object:Gem::Requirement
@@ -556,20 +542,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-- !ruby/object:Gem::Dependency
-  name: http
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '5.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '5.0'
 - !ruby/object:Gem::Dependency
   name: jr-cli
   requirement: !ruby/object:Gem::Requirement
@@ -850,6 +822,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: semantic_logger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.10'
+- !ruby/object:Gem::Dependency
+  name: sentry-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.1.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.1.1
 - !ruby/object:Gem::Dependency
   name: shodanx
   requirement: !ruby/object:Gem::Requirement
@@ -1080,7 +1080,6 @@ files:
 - lib/mihari/cli/base.rb
 - lib/mihari/cli/init.rb
 - lib/mihari/cli/main.rb
-- lib/mihari/cli/mixins/utils.rb
 - lib/mihari/cli/validator.rb
 - lib/mihari/commands/init.rb
 - lib/mihari/commands/search.rb
@@ -1092,18 +1091,33 @@ files:
 - lib/mihari/emitters/database.rb
 - lib/mihari/emitters/misp.rb
 - lib/mihari/emitters/slack.rb
-- lib/mihari/emitters/stdout.rb
 - lib/mihari/emitters/the_hive.rb
 - lib/mihari/emitters/webhook.rb
 - lib/mihari/enrichers/base.rb
 - lib/mihari/enrichers/ipinfo.rb
+- lib/mihari/entities/alert.rb
+- lib/mihari/entities/artifact.rb
+- lib/mihari/entities/autonomous_system.rb
+- lib/mihari/entities/command.rb
+- lib/mihari/entities/config.rb
+- lib/mihari/entities/dns.rb
+- lib/mihari/entities/geolocation.rb
+- lib/mihari/entities/ip_address.rb
+- lib/mihari/entities/message.rb
+- lib/mihari/entities/reverse_dns.rb
+- lib/mihari/entities/rule.rb
+- lib/mihari/entities/source.rb
+- lib/mihari/entities/tag.rb
+- lib/mihari/entities/whois.rb
 - lib/mihari/errors.rb
 - lib/mihari/feed/parser.rb
 - lib/mihari/feed/reader.rb
+- lib/mihari/http.rb
 - lib/mihari/mixins/autonomous_system.rb
 - lib/mihari/mixins/configurable.rb
 - lib/mihari/mixins/database.rb
 - lib/mihari/mixins/disallowed_data_value.rb
+- lib/mihari/mixins/error_notification.rb
 - lib/mihari/mixins/refang.rb
 - lib/mihari/mixins/retriable.rb
 - lib/mihari/mixins/rule.rb
@@ -1117,9 +1131,6 @@ files:
 - lib/mihari/models/tag.rb
 - lib/mihari/models/tagging.rb
 - lib/mihari/models/whois.rb
-- lib/mihari/notifiers/base.rb
-- lib/mihari/notifiers/exception_notifier.rb
-- lib/mihari/notifiers/slack.rb
 - lib/mihari/schemas/analyzer.rb
 - lib/mihari/schemas/macros.rb
 - lib/mihari/schemas/rule.rb
@@ -1147,36 +1158,29 @@ files:
 - lib/mihari/web/endpoints/rules.rb
 - lib/mihari/web/endpoints/sources.rb
 - lib/mihari/web/endpoints/tags.rb
-- lib/mihari/web/entities/alert.rb
-- lib/mihari/web/entities/artifact.rb
-- lib/mihari/web/entities/autonomous_system.rb
-- lib/mihari/web/entities/command.rb
-- lib/mihari/web/entities/config.rb
-- lib/mihari/web/entities/dns.rb
-- lib/mihari/web/entities/geolocation.rb
-- lib/mihari/web/entities/ip_address.rb
-- lib/mihari/web/entities/message.rb
-- lib/mihari/web/entities/reverse_dns.rb
-- lib/mihari/web/entities/rule.rb
-- lib/mihari/web/entities/source.rb
-- lib/mihari/web/entities/tag.rb
-- lib/mihari/web/entities/whois.rb
 - lib/mihari/web/middleware/connection_adapter.rb
+- lib/mihari/web/middleware/error_notification_adapter.rb
 - lib/mihari/web/public/index.html
 - lib/mihari/web/public/redoc-static.html
+- lib/mihari/web/public/static/css/app.43138058.css
+- lib/mihari/web/public/static/css/chunk-vendors.3ed9b08e.css
 - lib/mihari/web/public/static/favicon.ico
 - lib/mihari/web/public/static/fonts/fa-brands-400.099a9556.woff
 - lib/mihari/web/public/static/fonts/fa-brands-400.1a575a41.woff
+- lib/mihari/web/public/static/fonts/fa-brands-400.1fd0b4d7.ttf
 - lib/mihari/web/public/static/fonts/fa-brands-400.30cc681d.eot
 - lib/mihari/web/public/static/fonts/fa-brands-400.3b89dd10.ttf
 - lib/mihari/web/public/static/fonts/fa-brands-400.513aa607.ttf
 - lib/mihari/web/public/static/fonts/fa-brands-400.592643a8.eot
+- lib/mihari/web/public/static/fonts/fa-brands-400.5d5236fb.woff2
 - lib/mihari/web/public/static/fonts/fa-brands-400.ed311c7a.woff2
 - lib/mihari/web/public/static/fonts/fa-brands-400.f7307680.woff2
 - lib/mihari/web/public/static/fonts/fa-regular-400.1f77739c.ttf
+- lib/mihari/web/public/static/fonts/fa-regular-400.64b3730e.woff2
 - lib/mihari/web/public/static/fonts/fa-regular-400.7124eb50.woff
 - lib/mihari/web/public/static/fonts/fa-regular-400.7630483d.eot
 - lib/mihari/web/public/static/fonts/fa-regular-400.766913e6.ttf
+- lib/mihari/web/public/static/fonts/fa-regular-400.95a8a8af.ttf
 - lib/mihari/web/public/static/fonts/fa-regular-400.b0e2db3b.eot
 - lib/mihari/web/public/static/fonts/fa-regular-400.b91d376b.woff2
 - lib/mihari/web/public/static/fonts/fa-regular-400.d1d7e3b4.woff
@@ -1184,17 +1188,22 @@ files:
 - lib/mihari/web/public/static/fonts/fa-solid-900.0c6bfc66.eot
 - lib/mihari/web/public/static/fonts/fa-solid-900.1042e8ca.eot
 - lib/mihari/web/public/static/fonts/fa-solid-900.605ed792.ttf
+- lib/mihari/web/public/static/fonts/fa-solid-900.6115ad71.woff2
 - lib/mihari/web/public/static/fonts/fa-solid-900.9fe5a17c.woff
 - lib/mihari/web/public/static/fonts/fa-solid-900.b9625119.ttf
 - lib/mihari/web/public/static/fonts/fa-solid-900.d745348d.woff
 - lib/mihari/web/public/static/fonts/fa-solid-900.d824df7e.woff2
 - lib/mihari/web/public/static/fonts/fa-solid-900.e8a427e1.woff2
+- lib/mihari/web/public/static/fonts/fa-solid-900.f0203cfc.ttf
+- lib/mihari/web/public/static/fonts/fa-v4compatibility.e1023515.ttf
 - lib/mihari/web/public/static/img/fa-brands-400.1d5619cd.svg
 - lib/mihari/web/public/static/img/fa-brands-400.ba7ed552.svg
 - lib/mihari/web/public/static/img/fa-regular-400.0bb42845.svg
 - lib/mihari/web/public/static/img/fa-regular-400.c5d109be.svg
 - lib/mihari/web/public/static/img/fa-solid-900.376c1f97.svg
 - lib/mihari/web/public/static/img/fa-solid-900.37bc7099.svg
+- lib/mihari/web/public/static/js/app-legacy.46b666f0.js
+- lib/mihari/web/public/static/js/app-legacy.46b666f0.js.map
 - lib/mihari/web/public/static/js/app.06d5cf1c.js
 - lib/mihari/web/public/static/js/app.06d5cf1c.js.map
 - lib/mihari/web/public/static/js/app.0a0cc502.js
@@ -1205,6 +1214,8 @@ files:
 - lib/mihari/web/public/static/js/app.365f1907.js.map
 - lib/mihari/web/public/static/js/app.378da3dc.js
 - lib/mihari/web/public/static/js/app.378da3dc.js.map
+- lib/mihari/web/public/static/js/app.4818aedd.js
+- lib/mihari/web/public/static/js/app.4818aedd.js.map
 - lib/mihari/web/public/static/js/app.49ab738a.js
 - lib/mihari/web/public/static/js/app.49ab738a.js.map
 - lib/mihari/web/public/static/js/app.5dc97aae.js
@@ -1231,6 +1242,10 @@ files:
 - lib/mihari/web/public/static/js/app.f2b8890f.js.map
 - lib/mihari/web/public/static/js/app.fbc19869.js
 - lib/mihari/web/public/static/js/app.fbc19869.js.map
+- lib/mihari/web/public/static/js/chunk-vendors-legacy.c99e452e.js
+- lib/mihari/web/public/static/js/chunk-vendors-legacy.c99e452e.js.map
+- lib/mihari/web/public/static/js/chunk-vendors.15e84e22.js
+- lib/mihari/web/public/static/js/chunk-vendors.15e84e22.js.map
 - mihari.gemspec
 - renovate.json
 - sig/lib/mihari.rbs
@@ -1257,7 +1272,6 @@ files:
 - sig/lib/mihari/cli/base.rbs
 - sig/lib/mihari/cli/init.rbs
 - sig/lib/mihari/cli/main.rbs
-- sig/lib/mihari/cli/mixins/utils.rbs
 - sig/lib/mihari/cli/validator.rbs
 - sig/lib/mihari/commands/init.rbs
 - sig/lib/mihari/commands/json.rbs
@@ -1278,10 +1292,12 @@ files:
 - sig/lib/mihari/errors.rbs
 - sig/lib/mihari/feed/parser.rbs
 - sig/lib/mihari/feed/reader.rbs
+- sig/lib/mihari/http.rbs
 - sig/lib/mihari/mixins/autonomous_system.rbs
 - sig/lib/mihari/mixins/configurable.rbs
 - sig/lib/mihari/mixins/configuration.rbs
 - sig/lib/mihari/mixins/disallowed_data_value.rbs
+- sig/lib/mihari/mixins/error_notification.rbs
 - sig/lib/mihari/mixins/hash.rbs
 - sig/lib/mihari/mixins/refang.rbs
 - sig/lib/mihari/mixins/retriable.rbs
@@ -1296,9 +1312,6 @@ files:
 - sig/lib/mihari/models/tag.rbs
 - sig/lib/mihari/models/tagging.rbs
 - sig/lib/mihari/models/whois.rbs
-- sig/lib/mihari/notifiers/base.rbs
-- sig/lib/mihari/notifiers/exception_notifier.rbs
-- sig/lib/mihari/notifiers/slack.rbs
 - sig/lib/mihari/status.rbs
 - sig/lib/mihari/structs/alert.rbs
 - sig/lib/mihari/structs/censys.rbs