mihari 2.2.0 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c5859ab6b5d161359ea07c4af14de087013befa3b51d91844a60127137882e28
-  data.tar.gz: 481f34619d14e04f3187a9d2c3a05b3a1d109c7809622accebc47b100a506bc4
+  metadata.gz: 513324945758cadacf94345cb8a2a410cab99a08abc0ad02148c7737660d7348
+  data.tar.gz: 5c031467f3a6da288c5fceb137c17102654de072b6b34b2374dd15c523d09a03
 SHA512:
-  metadata.gz: 13693510f6f7d3560d207bb97dec50bb4851514e3b35918a82924e4e73d18e4bdcf963c3cd8928cd17559187b69419c8aa2a8e11e9d857965473dabfdf12ac59
-  data.tar.gz: 8786ad5973687848a89c8737eb92e52dd636e912b6af0ddf339dccccbff28c42f55e4111d1967b1d9a7b8aca07618788e82d98c24a56ca54102b36c84231d21e
+  metadata.gz: bf28354ab901254085d6e055e9c7a4989ce3d75ede0a6e978c009e69eb8f8a60a198e3fe505b82b9c5eddfa29b1866cc51da4bb8b41cc1727352d5bacd04e4d1
+  data.tar.gz: fa111a82851f3e695ce1b5c7a582b030459e3dac3598cff1b4d943f03fea8bdd000d21f17434e080f80109d2f90a19d154e4291efced316bf9e3c2d59880f747

data/.gitignore

@@ -57,3 +57,10 @@ Gemfile.lock
 
 # SQLite
 *.db
+
+# Config
+mihari.yml
+
+# Rule
+rule.yml
+!lib/mihari/templates/rule.yml

data/.overcommit.yml

@@ -0,0 +1,12 @@
+PreCommit:
+  BundleCheck:
+    enabled: true
+
+  RuboCop:
+    enabled: true
+    required_executable: bundle
+    command: ["bundle", "exec", "standardrb"]
+    on_warn: fail
+
+  YamlSyntax:
+    enabled: true

data/README.md

@@ -8,6 +8,8 @@
 
 ![img](https://github.com/ninoseki/mihari/raw/master/images/logo.png)
 
+[![](images/tines.png)](https://tines.io?utm_source=github&utm_medium=sponsorship&utm_campaign=ninoseki)
+
 Mihari is a framework for continuous OSINT based threat hunting.
 
 ## How it works
@@ -46,18 +48,14 @@ Mihari supports the following services by default.
 - [VirusTotal](http://virustotal.com)
 - [ZoomEye](https://zoomeye.org)
 
-See [Usage](https://github.com/ninoseki/mihari/wiki/Usage) for more information.
-
 ## Docs
 
-- [Requirements & Installation](https://github.com/ninoseki/mihari/wiki/Requirements-&-Installation)
-- [Usage](https://github.com/ninoseki/mihari/wiki/Usage)
-- [Built-in Web App](https://github.com/ninoseki/mihari/wiki/Built-in-Web-App)
-- [Configuration](https://github.com/ninoseki/mihari/wiki/Configuration)
-- [Custom Script](https://github.com/ninoseki/mihari/wiki/Custom-Script)
-- [Docker](https://github.com/ninoseki/mihari/wiki/Docker)
-- [GitHub Actions](https://github.com/ninoseki/mihari/wiki/GitHub-Actions)
+- [Mihari Knowledge Base](https://www.notion.so/Mihari-Knowledge-Base-266994ff61204428ba6cfcebe40b0bd1)
 
 ## License
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+## Acknowledgement
+
+Mihari is proudly supported by [Tines.io](https://tines.io?utm_source=github&utm_medium=sponsorship&utm_campaign=ninoseki), The SOAR Platform for Enterprise Security Teams.

data/exe/mihari

@@ -5,4 +5,4 @@ $LOAD_PATH.unshift("#{__dir__}/../lib")
 
 require "mihari"
 
-Mihari::CLI.start
+Mihari::CLI::Main.start

data/lib/mihari.rb

@@ -1,8 +1,56 @@
 # frozen_string_literal: true
 
+require "colorize"
+require "dry/configurable"
+require "dry/files"
 require "mem"
+require "yaml"
+
+# Mixins
+require "mihari/mixins/configurable"
+require "mihari/mixins/configuration"
+require "mihari/mixins/hash"
+require "mihari/mixins/refang"
+require "mihari/mixins/retriable"
+require "mihari/mixins/rule"
+
+def truthy?(value)
+  return true if value == "true"
+  return true if value == true
+
+  false
+end
 
 module Mihari
+  extend Dry::Configurable
+  extend Mixins::Configuration
+
+  setting :binaryedge_api_key, ENV["BINARYEDGE_API_KEY"]
+  setting :censys_id, ENV["CENSYS_ID"]
+  setting :censys_secret, ENV["CENSYS_SECRET"]
+  setting :circl_passive_password, ENV["CIRCL_PASSIVE_PASSWORD"]
+  setting :circl_passive_username, ENV["CIRCL_PASSIVE_USERNAME"]
+  setting :misp_api_endpoint, ENV["MISP_API_ENDPOINT"]
+  setting :misp_api_key, ENV["MISP_API_KEY"]
+  setting :onyphe_api_key, ENV["ONYPHE_API_KEY"]
+  setting :otx_api_key, ENV["OTX_API_KEY"]
+  setting :passivetotal_api_key, ENV["PASSIVETOTAL_API_KEY"]
+  setting :passivetotal_username, ENV["PASSIVETOTAL_USERNAME"]
+  setting :pulsedive_api_key, ENV["PULSEDIVE_API_KEY"]
+  setting :securitytrails_api_key, ENV["SECURITYTRAILS_API_KEY"]
+  setting :shodan_api_key, ENV["SHODAN_API_KEY"]
+  setting :slack_channel, ENV["SLACK_CHANNEL"]
+  setting :slack_webhook_url, ENV["SLACK_WEBHOOK_URL"]
+  setting :spyse_api_key, ENV["SPYSE_API_KEY"]
+  setting :thehive_api_endpoint, ENV["THEHIVE_API_ENDPOINT"]
+  setting :thehive_api_key, ENV["THEHIVE_API_KEY"]
+  setting :urlscan_api_key, ENV["URLSCAN_API_KEY"]
+  setting :virustotal_api_key, ENV["VIRUSTOTAL_API_KEY"]
+  setting :zoomeye_api_key, ENV["ZOOMEYE_API_KEY"]
+  setting :webhook_url, ENV["WEBHOOK_URL"]
+  setting(:webhook_use_json_body, ENV["WEBHOOK_USE_JSON_BODY"]) { |value| truthy?(value) }
+  setting :database, ENV["DATABASE"] || "mihari.db"
+
   class << self
     include Mem
 
@@ -15,31 +63,52 @@ module Mihari
       []
     end
     memoize :analyzers
+
+    #
+    # Load configuration from YAML file
+    #
+    # @param [String] path Path to YAML file
+    #
+    # @return [nil]
+    #
+    def load_config_from_yaml(path)
+      config = load_config(path)
+
+      # validate loaded yaml data
+      validate_config config
+
+      config.each do |key, value|
+        Mihari.config.send("#{key.downcase}=".to_sym, value)
+      end
+    end
   end
 end
 
 require "mihari/version"
 require "mihari/errors"
 
-require "mihari/config"
-
 require "mihari/database"
 require "mihari/type_checker"
 
+# Constraints
+require "mihari/constraints"
+
+# Schemas
+require "mihari/schemas/configuration"
+require "mihari/schemas/rule"
+
+# Models
 require "mihari/models/alert"
 require "mihari/models/artifact"
 require "mihari/models/tag"
 require "mihari/models/tagging"
 
+# Serializers
 require "mihari/serializers/alert"
 require "mihari/serializers/artifact"
 require "mihari/serializers/tag"
 
-require "mihari/html"
-
-require "mihari/configurable"
-require "mihari/retriable"
-
+# Analyzers
 require "mihari/analyzers/base"
 require "mihari/analyzers/basic"
 
@@ -53,7 +122,6 @@ require "mihari/analyzers/onyphe"
 require "mihari/analyzers/otx"
 require "mihari/analyzers/passivetotal"
 require "mihari/analyzers/pulsedive"
-require "mihari/analyzers/securitytrails_domain_feed"
 require "mihari/analyzers/securitytrails"
 require "mihari/analyzers/shodan"
 require "mihari/analyzers/spyse"
@@ -61,26 +129,32 @@ require "mihari/analyzers/urlscan"
 require "mihari/analyzers/virustotal"
 require "mihari/analyzers/zoomeye"
 
-require "mihari/analyzers/free_text"
-require "mihari/analyzers/http_hash"
-require "mihari/analyzers/passive_dns"
-require "mihari/analyzers/passive_ssl"
-require "mihari/analyzers/reverse_whois"
-require "mihari/analyzers/ssh_fingerprint"
+require "mihari/analyzers/rule"
 
+# Notifiers
 require "mihari/notifiers/base"
 require "mihari/notifiers/slack"
 require "mihari/notifiers/exception_notifier"
 
+# Emitters
 require "mihari/emitters/base"
 require "mihari/emitters/database"
 require "mihari/emitters/misp"
 require "mihari/emitters/slack"
 require "mihari/emitters/stdout"
 require "mihari/emitters/the_hive"
+require "mihari/emitters/webhook"
 
+# Status checker
 require "mihari/status"
 
+# Web app
 require "mihari/web/app"
 
-require "mihari/cli"
+# CLIs
+require "mihari/cli/base"
+
+require "mihari/cli/analyzer"
+require "mihari/cli/init"
+
+require "mihari/cli/main"

data/lib/mihari/analyzers/base.rb

@@ -1,16 +1,21 @@
 # frozen_string_literal: true
 
+require "dry-initializer"
 require "parallel"
 
 module Mihari
   module Analyzers
     class Base
-      include Configurable
-      include Retriable
+      extend Dry::Initializer
+
+      include Mixins::Configurable
+      include Mixins::Retriable
 
       attr_accessor :ignore_old_artifacts, :ignore_threshold
 
-      def initialize
+      def initialize(*args, **kwargs)
+        super
+
         @ignore_old_artifacts = false
         @ignore_threshold = 0
       end
@@ -30,6 +35,7 @@ module Mihari
         raise NotImplementedError, "You must implement #{self.class}##{__method__}"
       end
 
+      # @return [String]
       def source
         self.class.to_s.split("::").last
       end
@@ -39,6 +45,11 @@ module Mihari
         []
       end
 
+      #
+      # Set artifacts & run emitters in parallel
+      #
+      # @return [nil]
+      #
       def run
         set_unique_artifacts
 
@@ -47,6 +58,13 @@ module Mihari
         end
       end
 
+      #
+      # Run emitter
+      #
+      # @param [Mihari::Emitters::Base] emitter
+      #
+      # @return [nil]
+      #
       def run_emitter(emitter)
         emitter.run(title: title, description: description, artifacts: unique_artifacts, source: source, tags: tags)
       rescue StandardError => e
@@ -57,22 +75,40 @@ module Mihari
         Mihari.analyzers << child
       end
 
-      private
-
+      #
+      # Normalize artifacts
+      # - Uniquefy artifacts by native #uniq
+      # - Convert data (string) into an artifact
+      # - Reject an invalid artifact
+      #
       # @return [Array<Mihari::Artifact>]
+      #
       def normalized_artifacts
         @normalized_artifacts ||= artifacts.compact.uniq.sort.map do |artifact|
-          artifact.is_a?(Artifact) ? artifact : Artifact.new(data: artifact)
+          # No need to set data_type manually
+          # It is set automatically in #initialize
+          artifact.is_a?(Artifact) ? artifact : Artifact.new(data: artifact, source: source)
         end.select(&:valid?)
       end
 
+      private
+
+      #
+      # Uniquefy artifacts
+      #
       # @return [Array<Mihari::Artifact>]
+      #
       def unique_artifacts
         @unique_artifacts ||= normalized_artifacts.select do |artifact|
           artifact.unique?(ignore_old_artifacts: ignore_old_artifacts, ignore_threshold: ignore_threshold)
         end
       end
 
+      #
+      # Set unique artifacts
+      #
+      # @return [nil]
+      #
       def set_unique_artifacts
         retry_on_error { unique_artifacts }
       rescue ArgumentError => _e
@@ -80,11 +116,16 @@ module Mihari
         raise Error, "Please configure #{klass} API settings properly"
       end
 
+      #
+      # Select valid emitters
+      #
+      # @return [Array<Mihari::Emitters::Base>]
+      #
       def valid_emitters
-        @valid_emitters ||= Mihari.emitters.map do |klass|
+        @valid_emitters ||= Mihari.emitters.filter_map do |klass|
           emitter = klass.new
           emitter.valid? ? emitter : nil
-        end.compact
+        end
       end
     end
   end

data/lib/mihari/analyzers/basic.rb

@@ -3,8 +3,7 @@
 module Mihari
   module Analyzers
     class Basic < Base
-      attr_accessor :title
-      attr_reader :description, :artifacts, :source, :tags
+      attr_reader :title, :description, :artifacts, :source, :tags
 
       def initialize(title:, description:, artifacts:, source:, tags: [])
         super()

data/lib/mihari/analyzers/binaryedge.rb

@@ -5,16 +5,10 @@ require "binaryedge"
 module Mihari
   module Analyzers
     class BinaryEdge < Base
-      attr_reader :title, :description, :query, :tags
-
-      def initialize(query, title: nil, description: nil, tags: [])
-        super()
-
-        @query = query
-        @title = title || "BinaryEdge lookup"
-        @description = description || "query = #{query}"
-        @tags = tags
-      end
+      param :query
+      option :title, default: proc { "BinaryEdge search" }
+      option :description, default: proc { "query = #{query}" }
+      option :tags, default: proc { [] }
 
       def artifacts
         results = search
@@ -22,9 +16,9 @@ module Mihari
 
         results.map do |result|
           events = result["events"] || []
-          events.map do |event|
+          events.filter_map do |event|
             event.dig "target", "ip"
-          end.compact
+          end
         end.flatten.compact.uniq
       end
 
@@ -52,7 +46,7 @@ module Mihari
         responses
       end
 
-      def config_keys
+      def configuration_keys
         %w[binaryedge_api_key]
       end
 

data/lib/mihari/analyzers/censys.rb

@@ -1,87 +1,50 @@
 # frozen_string_literal: true
 
-require "censu"
+require "censysx"
 
 module Mihari
   module Analyzers
     class Censys < Base
-      attr_reader :title, :description, :query, :tags, :type
-
-      def initialize(query, title: nil, description: nil, tags: [], type: "ipv4")
-        super()
-
-        @query = query
-        @title = title || "Censys lookup"
-        @description = description || "query = #{query}"
-        @tags = tags
-        @type = type
-      end
+      param :query
+      option :title, default: proc { "Censys search" }
+      option :description, default: proc { "query = #{query}" }
+      option :tags, default: proc { [] }
 
       def artifacts
-        case type
-        when "ipv4"
-          ipv4_lookup
-        when "websites"
-          websites_lookup
-        when "certificates"
-          certificates_lookup
-        else
-          raise InvalidInputError, "#{type} type is not supported." unless valid_type?
-        end
+        search
       end
 
       private
 
-      def valid_type?
-        %w[ipv4 websites certificates].include? type
-      end
-
-      def normalize(domain)
-        return domain unless domain.start_with?("*.")
-
-        domain.sub("*.", "")
-      end
-
-      def ipv4_lookup
+      def search
         ipv4s = []
 
-        res = api.ipv4.search(query: query)
-        res.each_page do |page|
-          ipv4s << page.map(&:ip)
-        end
+        cursor = nil
+        loop do
+          response = api.search(query, cursor: cursor)
+          ipv4s << response_to_ipv4s(response)
 
-        ipv4s.flatten
-      end
-
-      def websites_lookup
-        domains = []
-
-        res = api.websites.search(query: query)
-        res.each_page do |page|
-          domains << page.map(&:domain)
+          links = response.dig("result", "links")
+          cursor = links["next"]
+          break if cursor == ""
         end
 
-        domains.flatten
+        ipv4s.flatten
       end
 
-      def certificates_lookup
-        domains = []
-
-        res = api.certificates.search(query: query)
-        res.each_page do |page|
-          page.each do |result|
-            subject_dn = result.subject_dn
-            names = subject_dn.scan(/CN=(.+)/).flatten.first
-            next unless names
-
-            domains << names.split(",").map { |domain| normalize(domain) }
-          end
-        end
-
-        domains.flatten
+      #
+      # Extract IPv4s from Censys search API response
+      #
+      # @param [Hash] response
+      #
+      # @return [Array<String>]
+      #
+      def response_to_ipv4s(response)
+        hits = response.dig("result", "hits") || []
+        hits.map { |hit| hit["ip"] }
       end
 
-      def config_keys
+      def configuration_keys
         %w[censys_id censys_secret]
       end