mihari 4.1.1 → 4.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 02dcb4c10888bb90fd38fc6ae879eb54d1eea6e06fbe3c7c4f662a76271e85bc
-  data.tar.gz: dd146a224d1b280f58ea71a566472a0dd80ad7e676703d6b6240c744ccad7244
+  metadata.gz: dc5b6ab7daa3030a0ef0778df2753247bde10b978be709102ecf1be60b672a9c
+  data.tar.gz: 1c1c283cf1e2989e9e94e0aa582567dd71b7900f5c5faa40a0ce3f1b327982a3
 SHA512:
-  metadata.gz: b23693c762329a79dddfa92699cd55c3d6a566e7753fea87d41a854871eab6ca408491000c7dd944396ac32bab2468e9c00bfed81335f5f5253c701033a53a5e
-  data.tar.gz: 3bc90185ef6a9684b05e9d7177a90aec54a73e5d0978e7d0f50c8f56a69e6040db1df567a98d7425ce2656e18e704d5f672fae1b44323c163f2ce913644493f2
+  metadata.gz: 918ee19022035b5f5e3db9a00318253803b1cc430b45676225af94861fe6dc6fe51343545d224e6094f09ad37dc003713fbbcb1777904b01205903e22d05bf23
+  data.tar.gz: c5ce99eb3d8e01b1b2a8ac51916afa172c1fecb55611a3b8aa76e686c72801beb11135ed1c8aed31e11b693b7467dbe513902f55e229c834bbfcb09460ba4202

data/.github/workflows/test.yml

@@ -1,9 +1,13 @@
 name: Ruby CI
 
-on: [pull_request]
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
 
 jobs:
-  build:
+  test:
     runs-on: ubuntu-latest
 
     services:
@@ -39,10 +43,10 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [2.7, "3.0"]
+        ruby: [2.7, "3.0", 3.1]
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Install dependencies
         run: |
@@ -65,3 +69,21 @@ jobs:
           DATABASE: mysql2://mysql:mysql@127.0.0.1:3306/test
         run: |
           bundle exec rake
+
+      - name: Coveralls Parallel
+        uses: coverallsapp/github-action@master
+        with:
+          github-token: ${{ secrets.github_token }}
+          flag-name: run-${{ matrix.ruby-version }}
+          parallel: true
+
+  coverage:
+    name: Coverage
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Coveralls Finished
+        uses: coverallsapp/github-action@master
+        with:
+          github-token: ${{ secrets.github_token }}
+          parallel-finished: true

data/README.md

@@ -9,7 +9,7 @@
 
 [![](images/tines.png)](https://tines.io?utm_source=github&utm_medium=sponsorship&utm_campaign=ninoseki)
 
-Mihari is a framework for continuous OSINT based threat hunting.
+Mihari is a tool for OSINT based threat hunting.
 
 ## How it works
 

data/lib/mihari/analyzers/base.rb

@@ -47,15 +47,23 @@ module Mihari
       #
       # Set artifacts & run emitters in parallel
       #
-      # @return [nil]
+      # @return [Mihari::Alert, nil]
       #
       def run
+        unless configured?
+          class_name = self.class.to_s.split("::").last
+          raise ConfigurationError, "#{class_name} is not configured correctly"
+        end
+
         with_db_connection do
           set_enriched_artifacts
 
-          Parallel.each(valid_emitters) do |emitter|
+          responses = Parallel.map(valid_emitters) do |emitter|
             run_emitter emitter
           end
+
+          # returns Mihari::Alert created by the database emitter
+          responses.find { |res| res.is_a?(Mihari::Alert) }
         end
       end
 
@@ -69,23 +77,26 @@ module Mihari
       def run_emitter(emitter)
         emitter.run(title: title, description: description, artifacts: enriched_artifacts, source: source, tags: tags)
       rescue StandardError => e
-        puts "Emission by #{emitter.class} is failed: #{e}"
+        Mihari.logger.info "Emission by #{emitter.class} is failed: #{e}"
       end
 
-      def self.inherited(child)
-        Mihari.analyzers << child
+      class << self
+        def inherited(child)
+          super
+          Mihari.analyzers << child
+        end
       end
 
       #
       # Normalize artifacts
-      # - Uniquefy artifacts by native #uniq
       # - Convert data (string) into an artifact
       # - Reject an invalid artifact
+      # - Uniquefy artifacts by data
       #
       # @return [Array<Mihari::Artifact>]
       #
       def normalized_artifacts
-        @normalized_artifacts ||= artifacts.compact.uniq.sort.map do |artifact|
+        @normalized_artifacts ||= artifacts.compact.sort.map do |artifact|
           # No need to set data_type manually
           # It is set automatically in #initialize
           artifact.is_a?(Artifact) ? artifact : Artifact.new(data: artifact, source: source)
@@ -124,9 +135,6 @@ module Mihari
       #
       def set_enriched_artifacts
         retry_on_error { enriched_artifacts }
-      rescue ArgumentError => e
-        klass = self.class.to_s.split("::").last.to_s
-        raise Error, "Please configure #{klass} settings properly. (#{e})"
       end
 
       #

data/lib/mihari/analyzers/rule.rb

@@ -28,6 +28,15 @@ module Mihari
       "zoomeye" => ZoomEye
     }.freeze
 
+    EMITTER_TO_CLASS = {
+      "database" => Emitters::Database,
+      "http" => Emitters::HTTP,
+      "misp" => Emitters::MISP,
+      "slack" => Emitters::Slack,
+      "the_hive" => Emitters::TheHive,
+      "webhook" => Emitters::Webhook
+    }.freeze
+
     class Rule < Base
       include Mixins::DisallowedDataValue
       include Mixins::Rule
@@ -41,6 +50,8 @@ module Mihari
       option :allowed_data_types, default: proc { ALLOWED_DATA_TYPES }
       option :disallowed_data_values, default: proc { [] }
 
+      option :emitters, optional: true
+
       attr_reader :source
 
       def initialize(**kwargs)
@@ -48,6 +59,8 @@ module Mihari
 
         @source = id
 
+        @emitters = emitters || DEFAULT_EMITTERS
+
         validate_analyzer_configurations
       end
 
@@ -59,18 +72,20 @@ module Mihari
       def artifacts
         artifacts = []
 
-        queries.each do |params|
-          analyzer_name = params[:analyzer]
+        queries.each do |original_params|
+          parmas = original_params.deep_dup
+
+          analyzer_name = parmas[:analyzer]
           klass = get_analyzer_class(analyzer_name)
 
-          query = params[:query]
+          query = parmas[:query]
 
           # set interval in the top level
-          options = params[:options] || {}
+          options = parmas[:options] || {}
           interval = options[:interval]
-          params[:interval] = interval if interval
+          parmas[:interval] = interval if interval
 
-          analyzer = klass.new(query, **params)
+          analyzer = klass.new(query, **parmas)
 
           # Use #normalized_artifacts method to get atrifacts as Array<Mihari::Artifact>
           # So Mihari::Artifact object has "source" attribute (e.g. "Shodan")
@@ -120,6 +135,34 @@ module Mihari
 
       private
 
+      #
+      # Get emitter class
+      #
+      # @param [String] emitter_name
+      #
+      # @return [Class<Mihari::Emitters::Base>] emitter class
+      #
+      def get_emitter_class(emitter_name)
+        emitter = EMITTER_TO_CLASS[emitter_name]
+        return emitter if emitter
+
+        raise ArgumentError, "#{emitter_name} is not supported"
+      end
+
+      def valid_emitters
+        @valid_emitters ||= emitters.filter_map do |original_params|
+          params = original_params.deep_dup
+
+          name = params[:emitter]
+          params.delete(:emitter)
+
+          klass = get_emitter_class(name)
+          emitter = klass.new(**params)
+
+          emitter.valid? ? emitter : nil
+        end
+      end
+
       #
       # Get analyzer class
       #
@@ -145,7 +188,7 @@ module Mihari
           instance = klass.new("dummy")
           unless instance.configured?
             klass_name = klass.to_s.split("::").last
-            raise ArgumentError, "#{klass_name} is not configured correctly"
+            raise ConfigurationError, "#{klass_name} is not configured correctly"
           end
         end
       end

data/lib/mihari/cli/base.rb

@@ -1,12 +1,8 @@
 # frozen_string_literal: true
 
-require "mihari/cli/mixins/utils"
-
 module Mihari
   module CLI
     class Base < Thor
-      include Mixins::Utils
-
       class << self
         def exit_on_failure?
           true

data/lib/mihari/commands/init.rb

@@ -19,7 +19,7 @@ module Mihari
 
             initialize_rule_yaml filename
 
-            puts "The rule file is initialized as #{filename}.".colorize(:blue)
+            Mihari.logger.info "The rule file is initialized as #{filename}."
           end
         end
       end

data/lib/mihari/commands/search.rb

@@ -5,12 +5,14 @@ module Mihari
     module Search
       include Mixins::Database
       include Mixins::Rule
+      include Mixins::ErrorNotification
 
       def self.included(thor)
         thor.class_eval do
           desc "search [RULE]", "Search by a rule"
           def search_by_rule(path_or_id)
             rule = load_rule(path_or_id)
+
             # validate
             begin
               validate_rule! rule
@@ -18,22 +20,17 @@ module Mihari
               raise e
             end
 
-            # build and run the analyzer
-            analyzer = build_rule_analyzer(
-              title: rule[:title],
-              description: rule[:description],
-              queries: rule[:queries],
-              tags: rule[:tags],
-              allowed_data_types: rule[:allowed_data_types],
-              disallowed_data_values: rule[:disallowed_data_values],
-              id: rule.id
-            )
+            analyzer = rule.to_analyzer
 
-            ignore_old_artifacts = rule[:ignore_old_artifacts]
-            ignore_threshold = rule[:ignore_threshold]
+            with_error_notification do
+              alert = analyzer.run
 
-            with_error_handling do
-              run_rule_analyzer analyzer, ignore_old_artifacts: ignore_old_artifacts, ignore_threshold: ignore_threshold
+              if alert
+                data = Mihari::Entities::Alert.represent(alert)
+                puts JSON.pretty_generate(data.as_json)
+              else
+                Mihari.logger.info "There is no new artifact"
+              end
 
               # record a rule
               with_db_connection do
@@ -46,50 +43,6 @@ module Mihari
           end
         end
       end
-
-      private
-
-      #
-      # Build a rule analyzer
-      #
-      # @param [String] title
-      # @param [String] description
-      # @param [Array<Hash>] queries
-      # @param [Array<String>, nil] tags
-      # @param [Array<String>, nil] allowed_data_types
-      # @param [Array<String>, nil] disallowed_data_values
-      #
-      # @return [Mihari::Analyzers::Rule]
-      #
-      def build_rule_analyzer(title:, description:, queries:, tags: nil, allowed_data_types: nil, disallowed_data_values: nil, id: nil)
-        tags = [] if tags.nil?
-        allowed_data_types = ALLOWED_DATA_TYPES if allowed_data_types.nil?
-        disallowed_data_values = [] if disallowed_data_values.nil?
-
-        Analyzers::Rule.new(
-          title: title,
-          description: description,
-          tags: tags,
-          queries: queries,
-          allowed_data_types: allowed_data_types,
-          disallowed_data_values: disallowed_data_values,
-          id: id
-        )
-      end
-
-      #
-      # Run rule analyzer
-      #
-      # @param [Mihari::Analyzer::Rule] analyzer
-      #
-      # @return [nil]
-      #
-      def run_rule_analyzer(analyzer, ignore_old_artifacts: false, ignore_threshold: 0)
-        analyzer.ignore_old_artifacts = ignore_old_artifacts
-        analyzer.ignore_threshold = ignore_threshold
-
-        analyzer.run
-      end
     end
   end
 end

data/lib/mihari/commands/validator.rb

@@ -13,8 +13,7 @@ module Mihari
 
             begin
               validate_rule! rule
-              puts "Valid format. The input is parsed as the following:"
-              puts rule.data.to_yaml
+              Mihari.logger.info "Valid format. The input is parsed as the following:\n#{rule.data.to_yaml}"
             rescue RuleValidationError
               nil
             end

data/lib/mihari/constants.rb

@@ -2,4 +2,6 @@
 
 module Mihari
   ALLOWED_DATA_TYPES = ["hash", "ip", "domain", "url", "mail"].freeze
+
+  DEFAULT_EMITTERS = ["database", "misp", "slack", "the_hive", "webhook"].map { |name| { emitter: name } }.freeze
 end

data/lib/mihari/emitters/base.rb

@@ -6,8 +6,14 @@ module Mihari
       include Mixins::Configurable
       include Mixins::Retriable
 
-      def self.inherited(child)
-        Mihari.emitters << child
+      def initialize(*)
+      end
+
+      class << self
+        def inherited(child)
+          super
+          Mihari.emitters << child
+        end
       end
 
       # @return [Boolean]

data/lib/mihari/emitters/http.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require "erb"
+
+module Mihari
+  module Emitters
+    class PayloadTemplate < ERB
+      def self.template
+        %{
+					{
+						"title": "<%= @title %>",
+						"description": "<%= @description %>",
+						"source": "<%= @source %>",
+						"artifacts": [
+							<% @artifacts.each_with_index do |artifact, idx| %>
+								"<%= artifact.data %>"
+								<%= ',' if idx < (@artifacts.length - 1) %>
+							<% end %>
+						],
+						"tags": [
+							<% @tags.each_with_index do |tag, idx| %>
+								"<%= tag %>"
+								<%= ',' if idx < (@tags.length - 1) %>
+							<% end %>
+						]
+					}
+				}
+      end
+
+      def initialize(title:, description:, artifacts:, source:, tags:, options: {})
+        @title = title
+        @description = description
+        @artifacts = artifacts
+        @source = source
+        @tags = tags
+
+        @template = options.fetch(:template, self.class.template)
+        super(@template)
+      end
+
+      def result
+        super(binding)
+      end
+    end
+
+    class HTTP < Base
+      # @return [Addressable::URI, nil]
+      attr_reader :uri
+
+      # @return [Hash]
+      attr_reader :http_request_headers
+
+      # @return [String]
+      attr_reader :http_request_method
+
+      # @return [String, nil]
+      attr_reader :template
+
+      def initialize(*args, **kwargs)
+        super(*args, **kwargs)
+
+        uri = kwargs[:url] || kwargs[:uri]
+        http_request_headers = kwargs[:http_request_headers] || {}
+        http_request_method = kwargs[:http_request_method] || "POST"
+        template = kwargs[:template]
+
+        @uri = Addressable::URI.parse(uri) if uri
+        @http_request_headers = http_request_headers
+        @http_request_method = http_request_method
+        @template = template
+      end
+
+      def emit(title:, description:, artifacts:, source:, tags:)
+        return if artifacts.empty?
+
+        res = nil
+
+        payload_ = payload_as_string(
+          title: title,
+          description: description,
+          artifacts: artifacts,
+          source: source,
+          tags: tags
+        )
+        payload = JSON.parse(payload_)
+
+        client = Mihari::HTTP.new(uri, headers: http_request_headers, payload: payload)
+
+        case http_request_method
+        when "GET"
+          res = client.get
+        when "POST"
+          res = client.post
+        end
+
+        res
+      end
+
+      def valid?
+        return false if uri.nil?
+
+        ["http", "https"].include? uri.scheme.downcase
+      end
+
+      private
+
+      def payload_as_string(title:, description:, artifacts:, source:, tags:)
+        @payload_as_string ||= [].tap do |out|
+          options = {}
+          unless template.nil?
+            options[:template] = File.read(template)
+          end
+
+          payload_template = PayloadTemplate.new(
+            title: title,
+            description: description,
+            artifacts: artifacts,
+            source: source,
+            tags: tags,
+            options: options
+          )
+          out << payload_template.result
+        end.first
+      end
+    end
+  end
+end

data/lib/mihari/emitters/slack.rb

@@ -109,12 +109,48 @@ module Mihari
     end
 
     class Slack < Base
-      def notifier
-        @notifier ||= Notifiers::Slack.new
+      SLACK_WEBHOOK_URL_KEY = "SLACK_WEBHOOK_URL"
+      SLACK_CHANNEL_KEY = "SLACK_CHANNEL"
+      DEFAULT_USERNAME = "mihari"
+
+      #
+      # Slack channel to post
+      #
+      # @return [String]
+      #
+      def slack_channel
+        Mihari.config.slack_channel || "#general"
+      end
+
+      #
+      # Slack webhook URL
+      #
+      # @return [String]
+      #
+      def slack_webhook_url
+        Mihari.config.slack_webhook_url
       end
 
+      #
+      # Check Slack webhook URL is set
+      #
+      # @return [Boolean]
+      #
+      def slack_webhook_url?
+        !Mihari.config.slack_webhook_url.nil?
+      end
+
+      #
+      # Check Slack webhook URL is set. Alias of #slack_webhook_url?.
+      #
+      # @return [Boolean]
+      #
       def valid?
-        notifier.valid?
+        slack_webhook_url?
+      end
+
+      def notifier
+        @notifier ||= ::Slack::Notifier.new(slack_webhook_url, channel: slack_channel, username: DEFAULT_USERNAME)
       end
 
       #
@@ -155,7 +191,7 @@ module Mihari
         attachments = to_attachments(artifacts)
         text = to_text(title: title, description: description, tags: tags)
 
-        notifier.notify(text: text, attachments: attachments)
+        notifier.post(text: text, attachments: attachments, mrkdwn: true)
       end
 
       private

data/lib/mihari/emitters/webhook.rb

@@ -11,20 +11,11 @@ module Mihari
       def emit(title:, description:, artifacts:, source:, tags:)
         return if artifacts.empty?
 
-        uri = Addressable::URI.parse(Mihari.config.webhook_url)
-        data = {
-          title: title,
-          description: description,
-          artifacts: artifacts.map(&:data),
-          source: source,
-          tags: tags
-        }
+        headers = { 'content-type': "application/x-www-form-urlencoded" }
+        headers["content-type"] = "application/json" if use_json_body?
 
-        if use_json_body?
-          Net::HTTP.post(uri, data.to_json, "Content-Type" => "application/json")
-        else
-          Net::HTTP.post_form(uri, data)
-        end
+        emitter = Emitters::HTTP.new(uri: Mihari.config.webhook_url)
+        emitter.emit(title: title, description: description, artifacts: artifacts, source: source, tags: tags)
       end
 
       private
@@ -45,16 +36,16 @@ module Mihari
       #
       # Check whether a webhook URL is set or not
       #
-      # @return [<Type>] <description>
+      # @return [Boolean]
       #
       def webhook_url?
         !webhook_url.nil?
       end
 
       #
-      # Check whether to use JSON body or NOT
+      # Check whether to use JSON body or not
       #
-      # @return [<Type>] <description>
+      # @return [Boolean]
       #
       def use_json_body?
         @use_json_body ||= Mihari.config.webhook_use_json_body

data/lib/mihari/enrichers/base.rb

@@ -5,8 +5,11 @@ module Mihari
     class Base
       include Mixins::Configurable
 
-      def self.inherited(child)
-        Mihari.enrichers << child
+      class << self
+        def inherited(child)
+          super
+          Mihari.enrichers << child
+        end
       end
 
       # @return [Boolean]

data/lib/mihari/enrichers/ipinfo.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require "http"
+require "net/https"
 
 module Mihari
   module Enrichers
@@ -34,11 +34,12 @@ module Mihari
           end
 
           begin
-            res = HTTP.headers(headers).get("https://ipinfo.io/#{ip}/json")
+            url = "https://ipinfo.io/#{ip}/json"
+            res = HTTP.get(url, headers: headers)
             data = JSON.parse(res.body.to_s)
 
             Structs::IPInfo::Response.from_dynamic! data
-          rescue HTTP::Error
+          rescue HttpError
             nil
           end
         end