mihari 4.1.0 → 4.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f96f1f1e70601518505d5b48aba5a9e0c60f1555e9d7c96ca307a32f5214a568
-  data.tar.gz: 87910483603ccf914b867bede2ea287d456c9bde59ccfebd2324c42b8b1c6928
+  metadata.gz: 853724da55a4225403595284e6437da9816d016d99e380d239507e1eecdf35cd
+  data.tar.gz: 35e5ef84d38c67eaa1670a5e6261aed5c629966921ca3b14fb2509c9e3993c45
 SHA512:
-  metadata.gz: 71fc241abdc3c41f28d49e7a2004957edaf0f09adc2f96b634af9dc5ea6de02fb2377fe0f88fbdb3c72816b2f72ebb4f6747e959a0dc7b13fff8377806bc57db
-  data.tar.gz: 9270f526e57e69df875f5a4cdacc0905cf8e8635f4fcea734f9a52b4f8b2e81a732a1990c4def0c53623ee57b60e7c40ee249a1b9501f866b1aec64fbbc1b47c
+  metadata.gz: 50855245bf70579da57c26859d2ab491e5238f7200c1de50cc7f46cafcd94c57e411e0fdbffdd60ddb00df2c8783ed051c87be24963fe10c5459784f337bd8d3
+  data.tar.gz: 436e6c0f3ceacb5b72f4212e83832759320cce109310736d87f6d1bfb49ac61f32a25a4d44cf133d2d5794466dea72bd69244997ccbbc6525418bc9aef403d0f

data/.github/workflows/test.yml

@@ -42,7 +42,7 @@ jobs:
         ruby: [2.7, "3.0"]
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Install dependencies
         run: |

data/README.md

@@ -9,7 +9,7 @@
 
 [![](images/tines.png)](https://tines.io?utm_source=github&utm_medium=sponsorship&utm_campaign=ninoseki)
 
-Mihari is a framework for continuous OSINT based threat hunting.
+Mihari is a tool for OSINT based threat hunting.
 
 ## How it works
 

data/Rakefile

@@ -7,3 +7,8 @@ require "standard/rake"
 RSpec::Core::RakeTask.new(:spec)
 
 task default: :spec
+
+desc "run rackup (via rerun)"
+task :rackup do
+  sh "rerun --pattern '{Gemfile.lock,lib/**/*.rb,lib/*.rb}' -- rackup config.ru"
+end

data/config.ru

@@ -1,4 +1,3 @@
-# bundle exec rerun -- rackup config.ru
 require "./lib/mihari"
 
 # set rack env as development

data/lib/mihari/analyzers/base.rb

@@ -47,15 +47,23 @@ module Mihari
       #
       # Set artifacts & run emitters in parallel
       #
-      # @return [nil]
+      # @return [Mihari::Alert, nil]
       #
       def run
+        unless configured?
+          class_name = self.class.to_s.split("::").last
+          raise ConfigurationError, "#{class_name} is not configured correctly"
+        end
+
         with_db_connection do
           set_enriched_artifacts
 
-          Parallel.each(valid_emitters) do |emitter|
+          responses = Parallel.map(valid_emitters) do |emitter|
             run_emitter emitter
           end
+
+          # returns Mihari::Alert created by the database emitter
+          responses.find { |res| res.is_a?(Mihari::Alert) }
         end
       end
 
@@ -69,23 +77,26 @@ module Mihari
       def run_emitter(emitter)
         emitter.run(title: title, description: description, artifacts: enriched_artifacts, source: source, tags: tags)
       rescue StandardError => e
-        puts "Emission by #{emitter.class} is failed: #{e}"
+        Mihari.logger.info "Emission by #{emitter.class} is failed: #{e}"
       end
 
-      def self.inherited(child)
-        Mihari.analyzers << child
+      class << self
+        def inherited(child)
+          super
+          Mihari.analyzers << child
+        end
       end
 
       #
       # Normalize artifacts
-      # - Uniquefy artifacts by native #uniq
       # - Convert data (string) into an artifact
       # - Reject an invalid artifact
+      # - Uniquefy artifacts by data
       #
       # @return [Array<Mihari::Artifact>]
       #
       def normalized_artifacts
-        @normalized_artifacts ||= artifacts.compact.uniq.sort.map do |artifact|
+        @normalized_artifacts ||= artifacts.compact.sort.map do |artifact|
           # No need to set data_type manually
           # It is set automatically in #initialize
           artifact.is_a?(Artifact) ? artifact : Artifact.new(data: artifact, source: source)
@@ -124,9 +135,6 @@ module Mihari
       #
       def set_enriched_artifacts
         retry_on_error { enriched_artifacts }
-      rescue ArgumentError => e
-        klass = self.class.to_s.split("::").last.to_s
-        raise Error, "Please configure #{klass} settings properly. (#{e})"
       end
 
       #

data/lib/mihari/analyzers/rule.rb

@@ -145,7 +145,7 @@ module Mihari
           instance = klass.new("dummy")
           unless instance.configured?
             klass_name = klass.to_s.split("::").last
-            raise ArgumentError, "#{klass_name} is not configured correctly"
+            raise ConfigurationError, "#{klass_name} is not configured correctly"
           end
         end
       end

data/lib/mihari/cli/base.rb

@@ -1,12 +1,8 @@
 # frozen_string_literal: true
 
-require "mihari/cli/mixins/utils"
-
 module Mihari
   module CLI
     class Base < Thor
-      include Mixins::Utils
-
       class << self
         def exit_on_failure?
           true

data/lib/mihari/commands/init.rb

@@ -19,7 +19,7 @@ module Mihari
 
             initialize_rule_yaml filename
 
-            puts "The rule file is initialized as #{filename}.".colorize(:blue)
+            Mihari.logger.info "The rule file is initialized as #{filename}."
           end
         end
       end

data/lib/mihari/commands/search.rb

@@ -5,12 +5,14 @@ module Mihari
     module Search
       include Mixins::Database
       include Mixins::Rule
+      include Mixins::ErrorNotification
 
       def self.included(thor)
         thor.class_eval do
           desc "search [RULE]", "Search by a rule"
           def search_by_rule(path_or_id)
             rule = load_rule(path_or_id)
+
             # validate
             begin
               validate_rule! rule
@@ -18,22 +20,17 @@ module Mihari
               raise e
             end
 
-            # build and run the analyzer
-            analyzer = build_rule_analyzer(
-              title: rule[:title],
-              description: rule[:description],
-              queries: rule[:queries],
-              tags: rule[:tags],
-              allowed_data_types: rule[:allowed_data_types],
-              disallowed_data_values: rule[:disallowed_data_values],
-              id: rule.id
-            )
+            analyzer = rule.to_analyzer
 
-            ignore_old_artifacts = rule[:ignore_old_artifacts]
-            ignore_threshold = rule[:ignore_threshold]
+            with_error_notification do
+              alert = analyzer.run
 
-            with_error_handling do
-              run_rule_analyzer analyzer, ignore_old_artifacts: ignore_old_artifacts, ignore_threshold: ignore_threshold
+              if alert
+                data = Mihari::Entities::Alert.represent(alert)
+                puts JSON.pretty_generate(data.as_json)
+              else
+                Mihari.logger.info "There is no new artifact"
+              end
 
               # record a rule
               with_db_connection do
@@ -46,50 +43,6 @@ module Mihari
           end
         end
       end
-
-      private
-
-      #
-      # Build a rule analyzer
-      #
-      # @param [String] title
-      # @param [String] description
-      # @param [Array<Hash>] queries
-      # @param [Array<String>, nil] tags
-      # @param [Array<String>, nil] allowed_data_types
-      # @param [Array<String>, nil] disallowed_data_values
-      #
-      # @return [Mihari::Analyzers::Rule]
-      #
-      def build_rule_analyzer(title:, description:, queries:, tags: nil, allowed_data_types: nil, disallowed_data_values: nil, id: nil)
-        tags = [] if tags.nil?
-        allowed_data_types = ALLOWED_DATA_TYPES if allowed_data_types.nil?
-        disallowed_data_values = [] if disallowed_data_values.nil?
-
-        Analyzers::Rule.new(
-          title: title,
-          description: description,
-          tags: tags,
-          queries: queries,
-          allowed_data_types: allowed_data_types,
-          disallowed_data_values: disallowed_data_values,
-          id: id
-        )
-      end
-
-      #
-      # Run rule analyzer
-      #
-      # @param [Mihari::Analyzer::Rule] analyzer
-      #
-      # @return [nil]
-      #
-      def run_rule_analyzer(analyzer, ignore_old_artifacts: false, ignore_threshold: 0)
-        analyzer.ignore_old_artifacts = ignore_old_artifacts
-        analyzer.ignore_threshold = ignore_threshold
-
-        analyzer.run
-      end
     end
   end
 end

data/lib/mihari/commands/validator.rb

@@ -13,8 +13,7 @@ module Mihari
 
             begin
               validate_rule! rule
-              puts "Valid format. The input is parsed as the following:"
-              puts rule.data.to_yaml
+              Mihari.logger.info "Valid format. The input is parsed as the following:\n#{rule.data.to_yaml}"
             rescue RuleValidationError
               nil
             end

data/lib/mihari/database.rb

@@ -173,7 +173,7 @@ module Mihari
       def close
         return unless ActiveRecord::Base.connected?
 
-        ActiveRecord::Base.clear_all_connections!
+        ActiveRecord::Base.clear_active_connections!
       end
 
       #

data/lib/mihari/emitters/base.rb

@@ -6,8 +6,11 @@ module Mihari
       include Mixins::Configurable
       include Mixins::Retriable
 
-      def self.inherited(child)
-        Mihari.emitters << child
+      class << self
+        def inherited(child)
+          super
+          Mihari.emitters << child
+        end
       end
 
       # @return [Boolean]

data/lib/mihari/emitters/slack.rb

@@ -109,12 +109,48 @@ module Mihari
     end
 
     class Slack < Base
-      def notifier
-        @notifier ||= Notifiers::Slack.new
+      SLACK_WEBHOOK_URL_KEY = "SLACK_WEBHOOK_URL"
+      SLACK_CHANNEL_KEY = "SLACK_CHANNEL"
+      DEFAULT_USERNAME = "mihari"
+
+      #
+      # Slack channel to post
+      #
+      # @return [String]
+      #
+      def slack_channel
+        Mihari.config.slack_channel || "#general"
+      end
+
+      #
+      # Slack webhook URL
+      #
+      # @return [String]
+      #
+      def slack_webhook_url
+        Mihari.config.slack_webhook_url
       end
 
+      #
+      # Check Slack webhook URL is set
+      #
+      # @return [Boolean]
+      #
+      def slack_webhook_url?
+        !Mihari.config.slack_webhook_url.nil?
+      end
+
+      #
+      # Check Slack webhook URL is set. Alias of #slack_webhook_url?.
+      #
+      # @return [Boolean]
+      #
       def valid?
-        notifier.valid?
+        slack_webhook_url?
+      end
+
+      def notifier
+        @notifier ||= ::Slack::Notifier.new(slack_webhook_url, channel: slack_channel, username: DEFAULT_USERNAME)
       end
 
       #
@@ -155,7 +191,7 @@ module Mihari
         attachments = to_attachments(artifacts)
         text = to_text(title: title, description: description, tags: tags)
 
-        notifier.notify(text: text, attachments: attachments)
+        notifier.post(text: text, attachments: attachments, mrkdwn: true)
       end
 
       private

data/lib/mihari/enrichers/base.rb

@@ -5,8 +5,11 @@ module Mihari
     class Base
       include Mixins::Configurable
 
-      def self.inherited(child)
-        Mihari.enrichers << child
+      class << self
+        def inherited(child)
+          super
+          Mihari.enrichers << child
+        end
       end
 
       # @return [Boolean]

data/lib/mihari/enrichers/ipinfo.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require "http"
+require "net/https"
 
 module Mihari
   module Enrichers
@@ -34,11 +34,12 @@ module Mihari
           end
 
           begin
-            res = HTTP.headers(headers).get("https://ipinfo.io/#{ip}/json")
+            url = "https://ipinfo.io/#{ip}/json"
+            res = HTTP.get(url, headers: headers)
             data = JSON.parse(res.body.to_s)
 
             Structs::IPInfo::Response.from_dynamic! data
-          rescue HTTP::Error
+          rescue HttpError
             nil
           end
         end

data/lib/mihari/errors.rb

@@ -14,4 +14,6 @@ module Mihari
   class FeedParseError < Error; end
 
   class RuleValidationError < Error; end
+
+  class ConfigurationError < Error; end
 end

data/lib/mihari/feed/reader.rb

@@ -18,29 +18,21 @@ module Mihari
       def read
         return read_file(uri.path) if uri.scheme == "file"
 
-        return get if http_request_method == "GET"
+        res = nil
+        client = HTTP.new(uri, headers: http_request_headers, payload: http_request_payload, payload_type: http_request_payload_type)
 
-        post
-      end
-
-      def get
-        uri.query = Addressable::URI.form_encode(http_request_payload)
-        get = Net::HTTP::Get.new(uri)
-
-        request(get)
-      end
+        res = client.get if http_request_method == "GET"
+        res = client.post if http_request_method == "POST"
 
-      def post
-        post = Net::HTTP::Post.new(uri)
+        return [] if res.nil?
 
-        case http_request_payload_type
-        when "application/json"
-          post.body = JSON.generate(http_request_payload)
-        when "application/x-www-form-urlencoded"
-          post.set_form_data(http_request_payload)
+        body = res.body
+        content_type = res["Content-Type"].to_s
+        if content_type.include?("application/json")
+          convert_as_json(body)
+        else
+          convert_as_csv(body)
         end
-
-        request(post)
       end
 
       #
@@ -70,42 +62,6 @@ module Mihari
         CSV.new(text_without_comments).to_a.reject(&:empty?)
       end
 
-      def https_options
-        return { use_ssl: true } if uri.scheme == "https"
-
-        {}
-      end
-
-      #
-      # Make a HTTP request
-      #
-      # @param [Net::HTTPRequest] req
-      #
-      # @return [Array<Hash>]
-      #
-      def request(req)
-        Net::HTTP.start(uri.host, uri.port, https_options) do |http|
-          # set headers
-          http_request_headers.each do |k, v|
-            req[k] = v
-          end
-
-          response = http.request(req)
-
-          code = response.code.to_i
-          raise HttpError, "Unsupported response code returned: #{code}" if code != 200
-
-          body = response.body
-
-          content_type = response["Content-Type"].to_s
-          if content_type.include?("application/json")
-            convert_as_json(body)
-          else
-            convert_as_csv(body)
-          end
-        end
-      end
-
       #
       # Read & convert a file
       #

data/lib/mihari/http.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module Mihari
+  class HTTP
+    attr_reader :uri, :headers, :payload_type, :payload
+
+    def initialize(uri, headers: {}, payload_type: nil, payload: {})
+      @uri = Addressable::URI.parse(uri)
+      @headers = headers
+      @payload_type = payload_type
+      @payload = payload
+    end
+
+    #
+    # Make a GET request
+    #
+    # @return [Net::HTTPResponse]
+    #
+    def get
+      uri.query = Addressable::URI.form_encode(payload)
+      get = Net::HTTP::Get.new(uri)
+
+      request(get)
+    end
+
+    #
+    # Make a POST request
+    #
+    # @return [Net::HTTPResponse]
+    #
+    def post
+      post = Net::HTTP::Post.new(uri)
+
+      case payload_type
+      when "application/json"
+        headers["content-type"] = "application/json" unless headers.key?("content-type")
+        post.body = JSON.generate(payload)
+      when "application/x-www-form-urlencoded"
+        headers["content-type"] = "application/x-www-form-urlencoded" unless headers.key?("content-type")
+        post.set_form_data(payload)
+      end
+
+      request(post)
+    end
+
+    class << self
+      def get(uri, headers: {}, payload_type: nil, payload: {})
+        client = new(uri, headers: headers, payload_type: payload_type, payload: payload)
+        client.get
+      end
+
+      def post(uri, headers: {}, payload_type: nil, payload: {})
+        client = new(uri, headers: headers, payload_type: payload_type, payload: payload)
+        client.post
+      end
+    end
+
+    private
+
+    #
+    # Get options for HTTP request
+    #
+    # @return [Hahs]
+    #
+    def https_options
+      return { use_ssl: true } if uri.scheme == "https"
+
+      {}
+    end
+
+    #
+    # Make a HTTP request
+    #
+    # @param [Net::HTTPRequest] req
+    #
+    # @return [Net::HTTPResponse]
+    #
+    def request(req)
+      Net::HTTP.start(uri.host, uri.port, https_options) do |http|
+        # set headers
+        headers.each do |k, v|
+          req[k] = v
+        end
+
+        res = http.request(req)
+
+        code = res.code.to_i
+        raise HttpError, "Unsupported response code returned: #{code}" if code != 200
+
+        res
+      end
+    end
+  end
+end

data/lib/mihari/mixins/error_notification.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Mixins
+    module ErrorNotification
+      #
+      # Send an exception notification if there is any error in a block
+      #
+      # @return [Nil]
+      #
+      def with_error_notification
+        yield
+      rescue StandardError => e
+        Mihari.logger.error e
+
+        Sentry.capture_exception(e) if Sentry.initialized?
+      end
+    end
+  end
+end

data/lib/mihari/mixins/retriable.rb

@@ -3,20 +3,30 @@
 module Mihari
   module Mixins
     module Retriable
+      DEFAULT_ON = [
+        Errno::ECONNRESET,
+        Errno::ECONNABORTED,
+        Errno::EPIPE,
+        OpenSSL::SSL::SSLError,
+        Timeout::Error,
+        RetryableError
+      ]
+
       #
       # Retry on error
       #
       # @param [Integer] times
       # @param [Integer] interval
+      # @param [Array<StandardError>] on
       #
       # @return [nil]
       #
-      def retry_on_error(times: 3, interval: 10)
+      def retry_on_error(times: 3, interval: 5, on: DEFAULT_ON)
         try = 0
         begin
           try += 1
           yield
-        rescue Errno::ECONNRESET, Errno::ECONNABORTED, Errno::EPIPE, OpenSSL::SSL::SSLError, Timeout::Error, RetryableError => e
+        rescue *on => e
           sleep interval
           retry if try < times
           raise e

data/lib/mihari/mixins/rule.rb

@@ -48,8 +48,7 @@ module Mihari
       def validate_rule!(rule)
         rule.validate!
       rescue RuleValidationError => e
-        error_message = "Failed to parse the input as a rule:\n#{e.message}"
-        puts error_message.colorize(:red)
+        Mihari.logger.error "Failed to parse the input as a rule"
         raise e
       end