mihari 2.4.0 → 3.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 388f48a9001d38fd83f4a6d527d7c5826a490be1d339596d174a9f619a78cb0c
-  data.tar.gz: 7e0a6e2fcbe9ad1792c21472b8be7706a86b09fbbce951edb1829a76493aaedc
+  metadata.gz: eeee7ec511b59cc1e6d07f47df8b2edc29860dbeeeafebc79ba876efe17f2954
+  data.tar.gz: e3c4fe0b49b20efa5ebdafaecdadbfb35135cbbdf3ff41dd36c68d2eaae644f8
 SHA512:
-  metadata.gz: dfcde6c4fa80ae12c56606157c6800c7e321cef71ed3e4aa9250805ea51126c74a19b3f73040630d169966fb17d834d8c45b37cc6f7baa808d7eea3e7c585fb9
-  data.tar.gz: d477cdcc4b4075e7671263f32ed5e81daad42e499eded5dffcecfba2d7568b779e99010a64a271a652db3f928800506cb6a4c7cf4060816742d4bb8d5bbec86a
+  metadata.gz: eff2de53ad20576849a81a5ebe96155c6b8747ab4b1775b26816ae6eff7eeb8324f2ad360755e867741a1b6cc778f1dec63d80bbb5439b9e8894536988971183
+  data.tar.gz: 2b1effec63ad4119f7cb521db866e0e9deb2dca7971036530d4cb16ea4a8669d591981ad5517784da13b98b7a3d3aa732538d040b53590b371d8f51acde849e0

data/.gitignore

@@ -57,3 +57,10 @@ Gemfile.lock
 
 # SQLite
 *.db
+
+# Config
+mihari.yml
+
+# Rule
+rule.yml
+!lib/mihari/templates/rule.yml

data/.overcommit.yml

@@ -0,0 +1,12 @@
+PreCommit:
+  BundleCheck:
+    enabled: true
+
+  RuboCop:
+    enabled: true
+    required_executable: bundle
+    command: ["bundle", "exec", "standardrb"]
+    on_warn: fail
+
+  YamlSyntax:
+    enabled: true

data/README.md

@@ -48,17 +48,9 @@ Mihari supports the following services by default.
 - [VirusTotal](http://virustotal.com)
 - [ZoomEye](https://zoomeye.org)
 
-See [Usage](https://github.com/ninoseki/mihari/wiki/Usage) for more information.
-
 ## Docs
 
-- [Requirements & Installation](https://github.com/ninoseki/mihari/wiki/Requirements-&-Installation)
-- [Usage](https://github.com/ninoseki/mihari/wiki/Usage)
-- [Built-in Web App](https://github.com/ninoseki/mihari/wiki/Built-in-Web-App)
-- [Configuration](https://github.com/ninoseki/mihari/wiki/Configuration)
-- [Custom Script](https://github.com/ninoseki/mihari/wiki/Custom-Script)
-- [Docker](https://github.com/ninoseki/mihari/wiki/Docker)
-- [GitHub Actions](https://github.com/ninoseki/mihari/wiki/GitHub-Actions)
+- [Mihari Knowledge Base](https://www.notion.so/Mihari-Knowledge-Base-266994ff61204428ba6cfcebe40b0bd1)
 
 ## License
 

data/build_frontend.sh

@@ -2,6 +2,7 @@
 
 CURRENT_DIR=${PWD}
 
+# build the frontend app
 mkdir -p tmp
 cd tmp
 git clone https://github.com/ninoseki/mihari-frontend.git
@@ -11,4 +12,8 @@ npm run build
 
 cp -r dist/* ${CURRENT_DIR}/lib/mihari/web/public
 
+# replace favicon path
+sed -i "" 's/href="\/favicon.ico"/href="\/static\/favicon.ico"/' ${CURRENT_DIR}/lib/mihari/web/public/index.html
+
+# remove tmp dir
 rm -rf ${CURRENT_DIR}/tmp/mihari-frontend

data/docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM ruby:3.0.1-alpine3.13
+FROM ruby:3.0.2-alpine3.13
 
 RUN apk --no-cache add git build-base ruby-dev sqlite-dev postgresql-dev mysql-client mysql-dev \
   && gem install pg mysql2 \

data/exe/mihari

@@ -5,4 +5,4 @@ $LOAD_PATH.unshift("#{__dir__}/../lib")
 
 require "mihari"
 
-Mihari::CLI.start
+Mihari::CLI::Main.start

data/lib/mihari.rb

@@ -1,8 +1,56 @@
 # frozen_string_literal: true
 
+require "colorize"
+require "dry/configurable"
+require "dry/files"
 require "mem"
+require "yaml"
+
+# Mixins
+require "mihari/mixins/configurable"
+require "mihari/mixins/configuration"
+require "mihari/mixins/hash"
+require "mihari/mixins/refang"
+require "mihari/mixins/retriable"
+require "mihari/mixins/rule"
+
+def truthy?(value)
+  return true if value == "true"
+  return true if value == true
+
+  false
+end
 
 module Mihari
+  extend Dry::Configurable
+  extend Mixins::Configuration
+
+  setting :binaryedge_api_key, ENV["BINARYEDGE_API_KEY"]
+  setting :censys_id, ENV["CENSYS_ID"]
+  setting :censys_secret, ENV["CENSYS_SECRET"]
+  setting :circl_passive_password, ENV["CIRCL_PASSIVE_PASSWORD"]
+  setting :circl_passive_username, ENV["CIRCL_PASSIVE_USERNAME"]
+  setting :misp_api_endpoint, ENV["MISP_API_ENDPOINT"]
+  setting :misp_api_key, ENV["MISP_API_KEY"]
+  setting :onyphe_api_key, ENV["ONYPHE_API_KEY"]
+  setting :otx_api_key, ENV["OTX_API_KEY"]
+  setting :passivetotal_api_key, ENV["PASSIVETOTAL_API_KEY"]
+  setting :passivetotal_username, ENV["PASSIVETOTAL_USERNAME"]
+  setting :pulsedive_api_key, ENV["PULSEDIVE_API_KEY"]
+  setting :securitytrails_api_key, ENV["SECURITYTRAILS_API_KEY"]
+  setting :shodan_api_key, ENV["SHODAN_API_KEY"]
+  setting :slack_channel, ENV["SLACK_CHANNEL"]
+  setting :slack_webhook_url, ENV["SLACK_WEBHOOK_URL"]
+  setting :spyse_api_key, ENV["SPYSE_API_KEY"]
+  setting :thehive_api_endpoint, ENV["THEHIVE_API_ENDPOINT"]
+  setting :thehive_api_key, ENV["THEHIVE_API_KEY"]
+  setting :urlscan_api_key, ENV["URLSCAN_API_KEY"]
+  setting :virustotal_api_key, ENV["VIRUSTOTAL_API_KEY"]
+  setting :zoomeye_api_key, ENV["ZOOMEYE_API_KEY"]
+  setting :webhook_url, ENV["WEBHOOK_URL"]
+  setting(:webhook_use_json_body, ENV["WEBHOOK_USE_JSON_BODY"]) { |value| truthy?(value) }
+  setting :database, ENV["DATABASE"] || "mihari.db"
+
   class << self
     include Mem
 
@@ -15,31 +63,53 @@ module Mihari
       []
     end
     memoize :analyzers
+
+    #
+    # Load configuration from YAML file
+    #
+    # @param [String] path Path to YAML file
+    #
+    # @return [nil]
+    #
+    def load_config_from_yaml(path)
+      config = load_config(path)
+
+      # validate loaded yaml data
+      validate_config config
+
+      config.each do |key, value|
+        Mihari.config.send("#{key.downcase}=".to_sym, value)
+      end
+    end
   end
 end
 
 require "mihari/version"
 require "mihari/errors"
 
-require "mihari/config"
-
 require "mihari/database"
 require "mihari/type_checker"
 
+# Constraints
+require "mihari/constraints"
+
+# Schemas
+require "mihari/schemas/analyzer"
+require "mihari/schemas/configuration"
+require "mihari/schemas/rule"
+
+# Models
 require "mihari/models/alert"
 require "mihari/models/artifact"
 require "mihari/models/tag"
 require "mihari/models/tagging"
 
+# Serializers
 require "mihari/serializers/alert"
 require "mihari/serializers/artifact"
 require "mihari/serializers/tag"
 
-require "mihari/html"
-
-require "mihari/configurable"
-require "mihari/retriable"
-
+# Analyzers
 require "mihari/analyzers/base"
 require "mihari/analyzers/basic"
 
@@ -53,7 +123,6 @@ require "mihari/analyzers/onyphe"
 require "mihari/analyzers/otx"
 require "mihari/analyzers/passivetotal"
 require "mihari/analyzers/pulsedive"
-require "mihari/analyzers/securitytrails_domain_feed"
 require "mihari/analyzers/securitytrails"
 require "mihari/analyzers/shodan"
 require "mihari/analyzers/spyse"
@@ -61,17 +130,14 @@ require "mihari/analyzers/urlscan"
 require "mihari/analyzers/virustotal"
 require "mihari/analyzers/zoomeye"
 
-require "mihari/analyzers/free_text"
-require "mihari/analyzers/http_hash"
-require "mihari/analyzers/passive_dns"
-require "mihari/analyzers/passive_ssl"
-require "mihari/analyzers/reverse_whois"
-require "mihari/analyzers/ssh_fingerprint"
+require "mihari/analyzers/rule"
 
+# Notifiers
 require "mihari/notifiers/base"
 require "mihari/notifiers/slack"
 require "mihari/notifiers/exception_notifier"
 
+# Emitters
 require "mihari/emitters/base"
 require "mihari/emitters/database"
 require "mihari/emitters/misp"
@@ -80,8 +146,16 @@ require "mihari/emitters/stdout"
 require "mihari/emitters/the_hive"
 require "mihari/emitters/webhook"
 
+# Status checker
 require "mihari/status"
 
+# Web app
 require "mihari/web/app"
 
-require "mihari/cli"
+# CLIs
+require "mihari/cli/base"
+
+require "mihari/cli/analyzer"
+require "mihari/cli/init"
+
+require "mihari/cli/main"

data/lib/mihari/analyzers/base.rb

@@ -1,16 +1,21 @@
 # frozen_string_literal: true
 
+require "dry-initializer"
 require "parallel"
 
 module Mihari
   module Analyzers
     class Base
-      include Configurable
-      include Retriable
+      extend Dry::Initializer
+
+      include Mixins::Configurable
+      include Mixins::Retriable
 
       attr_accessor :ignore_old_artifacts, :ignore_threshold
 
-      def initialize
+      def initialize(*args, **kwargs)
+        super
+
         @ignore_old_artifacts = false
         @ignore_threshold = 0
       end
@@ -30,6 +35,7 @@ module Mihari
         raise NotImplementedError, "You must implement #{self.class}##{__method__}"
       end
 
+      # @return [String]
       def source
         self.class.to_s.split("::").last
       end
@@ -39,6 +45,11 @@ module Mihari
         []
       end
 
+      #
+      # Set artifacts & run emitters in parallel
+      #
+      # @return [nil]
+      #
       def run
         set_unique_artifacts
 
@@ -47,6 +58,13 @@ module Mihari
         end
       end
 
+      #
+      # Run emitter
+      #
+      # @param [Mihari::Emitters::Base] emitter
+      #
+      # @return [nil]
+      #
       def run_emitter(emitter)
         emitter.run(title: title, description: description, artifacts: unique_artifacts, source: source, tags: tags)
       rescue StandardError => e
@@ -57,22 +75,40 @@ module Mihari
         Mihari.analyzers << child
       end
 
-      private
-
+      #
+      # Normalize artifacts
+      # - Uniquefy artifacts by native #uniq
+      # - Convert data (string) into an artifact
+      # - Reject an invalid artifact
+      #
       # @return [Array<Mihari::Artifact>]
+      #
       def normalized_artifacts
         @normalized_artifacts ||= artifacts.compact.uniq.sort.map do |artifact|
-          artifact.is_a?(Artifact) ? artifact : Artifact.new(data: artifact)
+          # No need to set data_type manually
+          # It is set automatically in #initialize
+          artifact.is_a?(Artifact) ? artifact : Artifact.new(data: artifact, source: source)
         end.select(&:valid?)
       end
 
+      private
+
+      #
+      # Uniquefy artifacts
+      #
       # @return [Array<Mihari::Artifact>]
+      #
       def unique_artifacts
         @unique_artifacts ||= normalized_artifacts.select do |artifact|
           artifact.unique?(ignore_old_artifacts: ignore_old_artifacts, ignore_threshold: ignore_threshold)
         end
       end
 
+      #
+      # Set unique artifacts
+      #
+      # @return [nil]
+      #
       def set_unique_artifacts
         retry_on_error { unique_artifacts }
       rescue ArgumentError => _e
@@ -80,11 +116,16 @@ module Mihari
         raise Error, "Please configure #{klass} API settings properly"
       end
 
+      #
+      # Select valid emitters
+      #
+      # @return [Array<Mihari::Emitters::Base>]
+      #
       def valid_emitters
-        @valid_emitters ||= Mihari.emitters.map do |klass|
+        @valid_emitters ||= Mihari.emitters.filter_map do |klass|
           emitter = klass.new
           emitter.valid? ? emitter : nil
-        end.compact
+        end
       end
     end
   end

data/lib/mihari/analyzers/basic.rb

@@ -3,8 +3,7 @@
 module Mihari
   module Analyzers
     class Basic < Base
-      attr_accessor :title
-      attr_reader :description, :artifacts, :source, :tags
+      attr_reader :title, :description, :artifacts, :source, :tags
 
       def initialize(title:, description:, artifacts:, source:, tags: [])
         super()

data/lib/mihari/analyzers/binaryedge.rb

@@ -5,16 +5,10 @@ require "binaryedge"
 module Mihari
   module Analyzers
     class BinaryEdge < Base
-      attr_reader :title, :description, :query, :tags
-
-      def initialize(query, title: nil, description: nil, tags: [])
-        super()
-
-        @query = query
-        @title = title || "BinaryEdge lookup"
-        @description = description || "query = #{query}"
-        @tags = tags
-      end
+      param :query
+      option :title, default: proc { "BinaryEdge search" }
+      option :description, default: proc { "query = #{query}" }
+      option :tags, default: proc { [] }
 
       def artifacts
         results = search
@@ -22,9 +16,9 @@ module Mihari
 
         results.map do |result|
           events = result["events"] || []
-          events.map do |event|
+          events.filter_map do |event|
             event.dig "target", "ip"
-          end.compact
+          end
         end.flatten.compact.uniq
       end
 
@@ -52,7 +46,7 @@ module Mihari
         responses
       end
 
-      def config_keys
+      def configuration_keys
         %w[binaryedge_api_key]
       end
 

data/lib/mihari/analyzers/censys.rb

@@ -1,87 +1,50 @@
 # frozen_string_literal: true
 
-require "censu"
+require "censysx"
 
 module Mihari
   module Analyzers
     class Censys < Base
-      attr_reader :title, :description, :query, :tags, :type
-
-      def initialize(query, title: nil, description: nil, tags: [], type: "ipv4")
-        super()
-
-        @query = query
-        @title = title || "Censys lookup"
-        @description = description || "query = #{query}"
-        @tags = tags
-        @type = type
-      end
+      param :query
+      option :title, default: proc { "Censys search" }
+      option :description, default: proc { "query = #{query}" }
+      option :tags, default: proc { [] }
 
       def artifacts
-        case type
-        when "ipv4"
-          ipv4_lookup
-        when "websites"
-          websites_lookup
-        when "certificates"
-          certificates_lookup
-        else
-          raise InvalidInputError, "#{type} type is not supported." unless valid_type?
-        end
+        search
       end
 
       private
 
-      def valid_type?
-        %w[ipv4 websites certificates].include? type
-      end
-
-      def normalize(domain)
-        return domain unless domain.start_with?("*.")
-
-        domain.sub("*.", "")
-      end
-
-      def ipv4_lookup
+      def search
         ipv4s = []
 
-        res = api.ipv4.search(query: query)
-        res.each_page do |page|
-          ipv4s << page.map(&:ip)
-        end
+        cursor = nil
+        loop do
+          response = api.search(query, cursor: cursor)
+          ipv4s << response_to_ipv4s(response)
 
-        ipv4s.flatten
-      end
-
-      def websites_lookup
-        domains = []
-
-        res = api.websites.search(query: query)
-        res.each_page do |page|
-          domains << page.map(&:domain)
+          links = response.dig("result", "links")
+          cursor = links["next"]
+          break if cursor == ""
         end
 
-        domains.flatten
+        ipv4s.flatten
       end
 
-      def certificates_lookup
-        domains = []
-
-        res = api.certificates.search(query: query)
-        res.each_page do |page|
-          page.each do |result|
-            subject_dn = result.subject_dn
-            names = subject_dn.scan(/CN=(.+)/).flatten.first
-            next unless names
-
-            domains << names.split(",").map { |domain| normalize(domain) }
-          end
-        end
-
-        domains.flatten
+      #
+      # Extract IPv4s from Censys search API response
+      #
+      # @param [Hash] response
+      #
+      # @return [Array<String>]
+      #
+      def response_to_ipv4s(response)
+        hits = response.dig("result", "hits") || []
+        hits.map { |hit| hit["ip"] }
       end
 
-      def config_keys
+      def configuration_keys
         %w[censys_id censys_secret]
       end