mihari 4.11.0 → 5.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eda042a7f3e0c70bb86a1008d24556b2daf4264142e01936e4bdab3da275ff39
-  data.tar.gz: a5e3711ecb0fb982ce6280fbed12261d753e2224966e3961ac2cfe2b83e0e15d
+  metadata.gz: 41784cbe3811a8f5f2a6a4de663ceaa03e2635c329c455b523e5fc55fda0c9e7
+  data.tar.gz: '052987fcb8805a8f71a0716d5e0970b0c4c7451059dded4b5149679ccda6c999'
 SHA512:
-  metadata.gz: f33e198db0d4964eb15372b30ef6733aaf3efe51b31640e62e33da2412089e30f618ea9509cc86d57a6e38ae65eac1dd5c7f3fab9c2dc1d5793d005bdcb95862
-  data.tar.gz: d7824b329d6117c2539e62044374b989538e4d22fd7641eb63b11db3b766ef52943f3559cb6a43be440b60f55609a763e316661e69196d6749486c8027cad5b2
+  metadata.gz: f16447e83adb4630baf9587eecea0dbb6f580de23894d70fa2c1729483faa6dc1c86fe5de2bd782100f8cba89e2a50e7ce2f0ab675de822fe5fb767fb2c6f45f
+  data.tar.gz: af70fcc510ed4fd6a434481a8372c56d20a54b605ee0b49425a1315531d2261db6df9588f1961ba34796728ff40f0ce188ad58e6c542f4c3644a8c8251b37eb5

data/.github/workflows/test.yml

@@ -43,7 +43,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [2.7, "3.0", 3.1]
+        ruby: [2.7, "3.0", 3.1, 3.2]
 
     steps:
       - uses: actions/checkout@v3

data/README.md

@@ -5,9 +5,19 @@
 [![Coverage Status](https://coveralls.io/repos/github/ninoseki/mihari/badge.svg?branch=master)](https://coveralls.io/github/ninoseki/mihari?branch=master)
 [![CodeFactor](https://www.codefactor.io/repository/github/ninoseki/mihari/badge)](https://www.codefactor.io/repository/github/ninoseki/mihari)
 
-![img](https://github.com/ninoseki/mihari/raw/master/images/logo.png)
-
-[![](images/tines.png)](https://tines.io?utm_source=github&utm_medium=sponsorship&utm_campaign=ninoseki)
+---
+
+<p align="center">
+  <img src="https://github.com/ninoseki/mihari/raw/master/images/logo.png"/>
+  <br/>
+  <a href="https://tines.io?utm_source=github&utm_medium=sponsorship&utm_campaign=ninoseki">
+    <img src="https://github.com/ninoseki/mihari/raw/master/images/Tines-Full_Logo-Tines_Black.png"/>
+  </a>
+  <br/>
+  Mihari is proudly supported by <a href="https://tines.io?utm_source=github&utm_medium=sponsorship&utm_campaign=ninoseki">Tines</a>
+</p>
+
+---
 
 Mihari is a tool for OSINT based threat hunting.
 

data/Steepfile

@@ -1,5 +1,4 @@
 target :lib do
-  signature "sig"
   check "lib"
 
   repo_path "vendor/rbs/gem_rbs_collection/gems"

data/build_frontend.sh

@@ -15,8 +15,5 @@ trash -r ${CURRENT_DIR}/lib/mihari/web/public/
 mkdir -p ${CURRENT_DIR}/lib/mihari/web/public/
 cp -r dist/* ${CURRENT_DIR}/lib/mihari/web/public
 
-# replace favicon path
-sed -i "" 's/href="\/favicon.ico"/href="\/static\/favicon.ico"/' ${CURRENT_DIR}/lib/mihari/web/public/index.html
-
 # remove tmp dir
 rm -rf ${CURRENT_DIR}/tmp/mihari-frontend

data/docker/Dockerfile

@@ -1,15 +1,14 @@
-FROM ruby:3.0.3-alpine3.13
-
-RUN apk --no-cache add git build-base ruby-dev sqlite-dev postgresql-dev mysql-client mysql-dev \
-  && gem install pg mysql2 \
-  && cd /tmp/ \
-  && git clone https://github.com/ninoseki/mihari.git \
-  && cd mihari \
-  && gem build mihari.gemspec -o mihari.gem \
-  && gem install mihari.gem \
-  && rm -rf /tmp/mihari \
-  && apk del --purge git build-base ruby-dev
+FROM ruby:3.1.3-alpine3.17
+
+RUN apk --no-cache add git build-base ruby-dev sqlite-dev postgresql-dev mysql-client mysql-dev && \
+  gem install pg mysql2
+
+ARG MIHARI_VERSION=4.11.0
+
+RUN gem install mihari -v ${MIHARI_VERSION}
+
+RUN apk del --purge git build-base ruby-dev
 
 ENTRYPOINT ["mihari"]
 
-CMD ["--help"]
+CMD ["--help"]

data/lib/mihari/analyzers/base.rb

@@ -5,18 +5,20 @@ module Mihari
     class Base
       extend Dry::Initializer
 
+      option :rule, default: proc {}
+
       include Mixins::AutonomousSystem
       include Mixins::Configurable
       include Mixins::Database
       include Mixins::Retriable
 
-      attr_accessor :ignore_old_artifacts, :ignore_threshold
+      # @return [Mihari::Structs::Rule, nil]
+      attr_reader :rule
 
       def initialize(*args, **kwargs)
-        super
+        super(*args, **kwargs)
 
-        @ignore_old_artifacts = false
-        @ignore_threshold = 0
+        @base_time = Time.now.utc
       end
 
       # @return [Array<String>, Array<Mihari::Artifact>]
@@ -24,26 +26,11 @@ module Mihari
         raise NotImplementedError, "You must implement #{self.class}##{__method__}"
       end
 
-      # @return [String]
-      def title
-        self.class.to_s.split("::").last.to_s
-      end
-
-      # @return [String]
-      def description
-        raise NotImplementedError, "You must implement #{self.class}##{__method__}"
-      end
-
       # @return [String]
       def source
         self.class.to_s.split("::").last.to_s
       end
 
-      # @return [Array<String>]
-      def tags
-        []
-      end
-
       #
       # Set artifacts & run emitters in parallel
       #
@@ -77,13 +64,7 @@ module Mihari
       def run_emitter(emitter)
         return if enriched_artifacts.empty?
 
-        alert_or_something = emitter.run(
-          title: title,
-          description: description,
-          artifacts: enriched_artifacts,
-          source: source,
-          tags: tags
-        )
+        alert_or_something = emitter.run(artifacts: enriched_artifacts, rule: rule)
 
         Mihari.logger.info "Emission by #{emitter.class} is succedded"
 
@@ -112,7 +93,10 @@ module Mihari
           # No need to set data_type manually
           # It is set automatically in #initialize
           artifact.is_a?(Artifact) ? artifact : Artifact.new(data: artifact, source: source)
-        end.select(&:valid?).uniq(&:data)
+        end.select(&:valid?).uniq(&:data).map do |artifact|
+          artifact.rule_id = rule&.id
+          artifact
+        end
       end
 
       private
@@ -124,7 +108,7 @@ module Mihari
       #
       def unique_artifacts
         @unique_artifacts ||= normalized_artifacts.select do |artifact|
-          artifact.unique?(ignore_old_artifacts: ignore_old_artifacts, ignore_threshold: ignore_threshold)
+          artifact.unique?(base_time: @base_time, artifact_lifetime: rule&.artifact_lifetime)
         end
       end
 

data/lib/mihari/analyzers/rule.rb

@@ -29,38 +29,21 @@ module Mihari
 
     EMITTER_TO_CLASS = {
       "database" => Emitters::Database,
-      "http" => Emitters::HTTP,
       "misp" => Emitters::MISP,
       "slack" => Emitters::Slack,
       "the_hive" => Emitters::TheHive,
       "webhook" => Emitters::Webhook
     }.freeze
 
-    class Rule < Base
-      include Mixins::DisallowedDataValue
-
-      option :title
-      option :description
-      option :queries
-
-      option :id, default: proc { "" }
-      option :tags, default: proc { [] }
-      option :allowed_data_types, default: proc { ALLOWED_DATA_TYPES }
-      option :disallowed_data_values, default: proc { [] }
+    # @return [Mihari::Structs::Rule]
+    attr_reader :rule
 
-      option :emitters, optional: true
-      option :enrichers, optional: true
-
-      attr_reader :source
+    class Rule < Base
+      include Mixins::FalsePositive
 
       def initialize(**kwargs)
         super(**kwargs)
 
-        @source = id
-
-        @emitters = emitters || DEFAULT_EMITTERS
-        @enrichers = enrichers || DEFAULT_ENRICHERS
-
         validate_analyzer_configurations
       end
 
@@ -72,7 +55,7 @@ module Mihari
       def artifacts
         artifacts = []
 
-        queries.each do |original_params|
+        rule.queries.each do |original_params|
           parmas = original_params.deep_dup
 
           analyzer_name = parmas[:analyzer]
@@ -83,8 +66,12 @@ module Mihari
           # set interval in the top level
           options = parmas[:options] || {}
           interval = options[:interval]
+
           parmas[:interval] = interval if interval
 
+          # set rule
+          parmas[:rule] = rule
+
           analyzer = klass.new(query, **parmas)
 
           # Use #normalized_artifacts method to get atrifacts as Array<Mihari::Artifact>
@@ -106,9 +93,9 @@ module Mihari
       #
       def normalized_artifacts
         @normalized_artifacts ||= artifacts.uniq(&:data).select(&:valid?).select do |artifact|
-          allowed_data_types.include? artifact.data_type
+          rule.data_types.include? artifact.data_type
         end.reject do |artifact|
-          disallowed_data_value? artifact.data
+          falsepositive? artifact.data
         end
       end
 
@@ -119,7 +106,7 @@ module Mihari
       #
       def enriched_artifacts
         @enriched_artifacts ||= Parallel.map(unique_artifacts) do |artifact|
-          enrichers.each do |enricher|
+          rule.enrichers.each do |enricher|
             artifact.enrich_by_enricher(enricher[:enricher])
           end
 
@@ -132,22 +119,22 @@ module Mihari
       #
       # @return [Array<Regexp, String>]
       #
-      def normalized_disallowed_data_values
-        @normalized_disallowed_data_values ||= disallowed_data_values.map { |v| normalize_disallowed_data_value v }
+      def normalized_falsepositives
+        @normalized_falsepositives ||= rule.falsepositives.map { |v| normalize_falsepositive v }
       end
 
       #
-      # Check whether a value is a disallowed data value or not
+      # Check whether a value is a falsepositive value or not
       #
       # @return [Boolean]
       #
-      def disallowed_data_value?(value)
-        return true if normalized_disallowed_data_values.include?(value)
+      def falsepositive?(value)
+        return true if normalized_falsepositives.include?(value)
 
-        normalized_disallowed_data_values.select do |disallowed_data_value|
-          disallowed_data_value.is_a?(Regexp)
-        end.any? do |disallowed_data_value|
-          disallowed_data_value.match?(value)
+        normalized_falsepositives.select do |falsepositive|
+          falsepositive.is_a?(Regexp)
+        end.any? do |falseposistive|
+          falseposistive.match?(value)
         end
       end
 
@@ -168,7 +155,7 @@ module Mihari
       end
 
       def valid_emitters
-        @valid_emitters ||= emitters.filter_map do |original_params|
+        @valid_emitters ||= rule.emitters.filter_map do |original_params|
           params = original_params.deep_dup
 
           name = params[:emitter]
@@ -199,7 +186,7 @@ module Mihari
       # Validate configuration of analyzers
       #
       def validate_analyzer_configurations
-        queries.each do |params|
+        rule.queries.each do |params|
           analyzer_name = params[:analyzer]
           klass = get_analyzer_class(analyzer_name)
 

data/lib/mihari/cli/main.rb

@@ -3,28 +3,23 @@
 require "thor"
 
 # Commands
-require "mihari/commands/search"
+require "mihari/commands/initializer"
+require "mihari/commands/searcher"
+require "mihari/commands/validator"
 require "mihari/commands/version"
 require "mihari/commands/web"
 
 # CLIs
 require "mihari/cli/base"
 
-require "mihari/cli/init"
-require "mihari/cli/validator"
-
 module Mihari
   module CLI
     class Main < Base
-      include Mihari::Commands::Search
+      include Mihari::Commands::Searcher
       include Mihari::Commands::Version
       include Mihari::Commands::Web
-
-      desc "init", "Sub commands to initialize a rule"
-      subcommand "init", Initialization
-
-      desc "validate", "Sub commands to validate format of a rule"
-      subcommand "validate", Validator
+      include Mihari::Commands::Validator
+      include Mihari::Commands::Initializer
     end
   end
 end

data/lib/mihari/commands/initializer.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require "pathname"
+
+module Mihari
+  module Commands
+    module Initializer
+      def self.included(thor)
+        thor.class_eval do
+          desc "init", "Initialize a new rule"
+          method_option :path, type: :string, default: "./rule.yml"
+          def init
+            path = options["path"]
+
+            warning = "#{path} exists. Do you want to overwrite it? (y/n)"
+            return if Pathname(path).exist? && !(yes? warning)
+
+            initialize_rule path
+
+            Mihari.logger.info "A new rule is initialized as #{path}."
+          end
+
+          no_commands do
+            #
+            # @return [Mihari::Structs::Rule]
+            #
+            def rule_template
+              Structs::Rule.from_path File.expand_path("../templates/rule.yml.erb", __dir__)
+            end
+
+            #
+            # Create a new rule
+            #
+            # @param [String] path
+            # @param [Dry::Files] files
+            #
+            # @return [nil]
+            #
+            def initialize_rule(path, files = Dry::Files.new)
+              files.write(path, rule_template.yaml)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mihari/commands/{search.rb → searcher.rb}

@@ -2,15 +2,15 @@
 
 module Mihari
   module Commands
-    module Search
+    module Searcher
       include Mixins::Database
       include Mixins::ErrorNotification
 
       def self.included(thor)
         thor.class_eval do
-          desc "search [RULE]", "Search by a rule"
+          desc "search [PATH]", "Search by a rule"
           method_option :yes, type: :boolean, aliases: "-y", desc: "yes to overwrite the rule in the database"
-          def search_by_rule(path_or_id)
+          def search(path_or_id)
             rule = Structs::Rule.from_path_or_id path_or_id
 
             # validate
@@ -21,38 +21,27 @@ module Mihari
             end
 
             # check update
-            id = rule.id
             yes = options["yes"] || false
             unless yes
               with_db_connection do
-                rule_ = Mihari::Rule.find(id)
-                next if rule.yaml == rule_.yaml
-                unless yes?("This operation will overwrite the rule in the database (Rule ID: #{id}). Are you sure you want to update the rule? (yes/no)")
+                next if Mihari::Rule.find(rule.id).data == rule.data.deep_stringify_keys
+                unless yes?("This operation will overwrite the rule in the database (Rule ID: #{rule.id}). Are you sure you want to update the rule? (y/n)")
                   return
                 end
               rescue ActiveRecord::RecordNotFound
                 next
               end
             end
-
-            analyzer = rule.to_analyzer
+            # update rule model
+            rule.model.save
 
             with_error_notification do
-              alert = analyzer.run
-
+              alert = rule.analyzer.run
               if alert
                 data = Mihari::Entities::Alert.represent(alert)
                 puts JSON.pretty_generate(data.as_json)
               else
-                Mihari.logger.info "No new alert created in the database"
-              end
-
-              # record a rule
-              with_db_connection do
-                model = rule.to_model
-                model.save
-              rescue ActiveRecord::RecordNotUnique
-                nil
+                Mihari.logger.info "There is no new alert created in the database"
               end
             end
           end

data/lib/mihari/commands/validator.rb

@@ -5,7 +5,7 @@ module Mihari
     module Validator
       def self.included(thor)
         thor.class_eval do
-          desc "rule [PATH]", "Validate rule file format"
+          desc "validate [PATH]", "Validate a rule file"
           #
           # Validate format of a rule
           #
@@ -13,7 +13,7 @@ module Mihari
           #
           # @return [nil]
           #
-          def rule(path)
+          def validate(path)
             rule = Structs::Rule.from_path_or_id(path)
 
             begin

data/lib/mihari/commands/web.rb

@@ -8,18 +8,20 @@ module Mihari
           desc "web", "Launch the web app"
           method_option :port, type: :numeric, default: 9292, desc: "Hostname to listen on"
           method_option :host, type: :string, default: "localhost", desc: "Port to listen on"
-          method_option :threads, type: :string, default: "1:1", desc: "min:max threads to use"
+          method_option :threads, type: :string, default: "0:5", desc: "min:max threads to use"
           method_option :verbose, type: :boolean, default: true, desc: "Report each request"
+          method_option :worker_timeout, type: :numeric, default: 60, desc: "Worker timeout value (in seconds)"
           def web
             port = options["port"]
             host = options["host"]
             threads = options["threads"]
             verbose = options["verbose"]
+            worker_timeout = options["worker_timeout"]
 
             # set rack env as production
             ENV["RACK_ENV"] ||= "production"
 
-            Mihari::App.run!(port: port, host: host, threads: threads, verbose: verbose)
+            Mihari::App.run!(port: port, host: host, threads: threads, verbose: verbose, worker_timeout: worker_timeout)
           end
         end
       end

data/lib/mihari/constants.rb

@@ -1,9 +1,9 @@
 # frozen_string_literal: true
 
 module Mihari
-  ALLOWED_DATA_TYPES = ["hash", "ip", "domain", "url", "mail"].freeze
+  DEFAULT_DATA_TYPES = %w[hash ip domain url mail].freeze
 
-  DEFAULT_EMITTERS = ["database", "misp", "slack", "the_hive", "webhook"].map { |name| { emitter: name } }.freeze
+  DEFAULT_EMITTERS = %w[database misp slack the_hive].map { |name| { emitter: name } }.freeze
 
-  DEFAULT_ENRICHERS = ["whois", "ipinfo", "shodan", "google_public_dns"].map { |name| { enricher: name } }.freeze
+  DEFAULT_ENRICHERS = %w[whois ipinfo shodan google_public_dns].map { |name| { enricher: name } }.freeze
 end