mihari 3.0.1 → 3.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e890d0871e67d6f67aaf432ca236152286e0b9035f26cc1320f150416580e59e
-  data.tar.gz: 071d117cd29cd73f29f00c78dd42637248efc1073e43ce994393a8a7efeb75a6
+  metadata.gz: 142df147927aee93e6c653b2eb29cca50f4ba11606d68f1302af21780d6f0dc5
+  data.tar.gz: 594f762c94e361cc53cab08b39abad5e3503555b51d93add3cbce744fcbff711
 SHA512:
-  metadata.gz: 1acbd69f2f744b9025da82b41e4c6dbeb42f29eb7563fc6a0b2c09a5c332bfcd7913cee0639540e92437672503e3770ff1051b252f79deae653bbaeafedda590
-  data.tar.gz: 64492cabe3e7102e9614c915cf857db96cd2f1d9931174604131c2069f08d2cb1afc6136dcb4dc29c659f0e2c0d7f2b54836b8d81a0cae7389b00e37fee91b24
+  metadata.gz: 8483d2d125e30f04bdaf74243877dd9a261511d7d857f6dba42c6ea54af5de95f63c23759d1937badfdb5ae75819b99e70270edb0fb2f466601b8eaf6912dc8e
+  data.tar.gz: 8ffca15aadc0bc783086dd11df9fd051933af31e6778fcd4a67ddf51af5532b452603526a0303eb7c1dfdb530636a6e91df8440210189af3dc0f2806a4e64218

data/README.md

@@ -14,11 +14,12 @@ Mihari is a framework for continuous OSINT based threat hunting.
 
 ## How it works
 
-![img](https://github.com/ninoseki/mihari/raw/master/images/overview.png)
+![img](https://github.com/ninoseki/mihari/raw/master/images/overview.jpg)
 
 - Mihari makes a query against Shodan, Censys, VirusTotal, SecurityTrails, etc. and extracts artifacts (IP addresses, domains, URLs or hashes).
-- Mihari checks whether a DB (SQLite3, PostgreSQL or MySQL) contains the artifacts or not.
+- Mihari checks whether the database (SQLite3, PostgreSQL or MySQL) contains the artifacts or not.
   - If it doesn't contain the artifacts:
+    - Mihari saves artifacts in the database.
     - Mihari creates an alert on TheHive.
     - Mihari sends a notification to Slack.
     - Mihari creates an event on MISP.
@@ -52,6 +53,10 @@ Mihari supports the following services by default.
 
 - [Mihari Knowledge Base](https://www.notion.so/Mihari-Knowledge-Base-266994ff61204428ba6cfcebe40b0bd1)
 
+## Presentations
+
+- [Adversary Infrastructure Tracking with Mihari](https://ninoseki.github.io/presentations/Adversary%20Infrastructure%20Tracking%20with%20Mihari.pdf)
+
 ## License
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/build_frontend.sh

@@ -2,6 +2,7 @@
 
 CURRENT_DIR=${PWD}
 
+# build the frontend app
 mkdir -p tmp
 cd tmp
 git clone https://github.com/ninoseki/mihari-frontend.git
@@ -11,4 +12,8 @@ npm run build
 
 cp -r dist/* ${CURRENT_DIR}/lib/mihari/web/public
 
+# replace favicon path
+sed -i "" 's/href="\/favicon.ico"/href="\/static\/favicon.ico"/' ${CURRENT_DIR}/lib/mihari/web/public/index.html
+
+# remove tmp dir
 rm -rf ${CURRENT_DIR}/tmp/mihari-frontend

data/config.ru

@@ -0,0 +1,6 @@
+require "./lib/mihari"
+
+# set rack env as development
+ENV["RACK_ENV"] ||= "development"
+
+run Mihari::App

data/docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM ruby:3.0.1-alpine3.13
+FROM ruby:3.0.2-alpine3.13
 
 RUN apk --no-cache add git build-base ruby-dev sqlite-dev postgresql-dev mysql-client mysql-dev \
   && gem install pg mysql2 \

data/lib/mihari.rb

@@ -9,6 +9,7 @@ require "yaml"
 # Mixins
 require "mihari/mixins/configurable"
 require "mihari/mixins/configuration"
+require "mihari/mixins/disallowed_data_value"
 require "mihari/mixins/hash"
 require "mihari/mixins/refang"
 require "mihari/mixins/retriable"
@@ -94,6 +95,7 @@ require "mihari/type_checker"
 require "mihari/constraints"
 
 # Schemas
+require "mihari/schemas/analyzer"
 require "mihari/schemas/configuration"
 require "mihari/schemas/rule"
 

data/lib/mihari/analyzers/rule.rb

@@ -2,11 +2,11 @@
 
 require "uuidtools"
 
-NIL = nil
-
 module Mihari
   module Analyzers
     class Rule < Base
+      include Mihari::Mixins::DisallowedDataValue
+
       option :title
       option :description
       option :queries
@@ -14,6 +14,7 @@ module Mihari
       option :id, default: proc {}
       option :tags, default: proc { [] }
       option :allowed_data_types, default: proc { ALLOWED_DATA_TYPES }
+      option :disallowed_data_values, default: proc { [] }
 
       attr_reader :source
 
@@ -70,12 +71,36 @@ module Mihari
       # - Uniquefy artifacts by #uniq(&:data)
       # - Reject an invalid artifact (for just in case)
       # - Select artifacts with allowed data types
+      # - Reject artifacts with disallowed data values
       #
       # @return [Array<Mihari::Artifact>]
       #
       def normalized_artifacts
         @normalized_artifacts ||= artifacts.uniq(&:data).select(&:valid?).select do |artifact|
           allowed_data_types.include? artifact.data_type
+        end.reject do |artifact|
+          disallowed_data_value? artifact.data
+        end
+      end
+
+      #
+      # Normalized disallowed data values
+      #
+      # @return [Array<Regexp, String>]
+      #
+      def normalized_disallowed_data_values
+        @normalized_disallowed_data_values ||= disallowed_data_values.map { |v| normalize_disallowed_data_value v }
+      end
+
+      #
+      # Check whether a value is a disallowed data value or not
+      #
+      # @return [Boolean]
+      #
+      def disallowed_data_value?(value)
+        normalized_disallowed_data_values.any? do |disallowed_data_value|
+          return value == disallowed_data_value if disallowed_data_value.is_a?(String)
+          return disallowed_data_value.match?(value) if disallowed_data_value.is_a?(Regexp)
         end
       end
 

data/lib/mihari/cli/analyzer.rb

@@ -22,6 +22,9 @@ require "mihari/commands/json"
 module Mihari
   module CLI
     class Analyzer < Base
+      class_option :ignore_old_artifacts, type: :boolean, default: false, desc: "Whether to ignore old artifacts from checking or not."
+      class_option :ignore_threshold, type: :numeric, default: 0, desc: "Number of days to define whether an artifact is old or not."
+
       include Mihari::Commands::BinaryEdge
       include Mihari::Commands::Censys
       include Mihari::Commands::CIRCL

data/lib/mihari/cli/base.rb

@@ -14,9 +14,6 @@ module Mihari
 
       class_option :config, type: :string, desc: "Path to the config file"
 
-      class_option :ignore_old_artifacts, type: :boolean, default: false, desc: "Whether to ignore old artifacts from checking or not. Only affects with analyze commands."
-      class_option :ignore_threshold, type: :numeric, default: 0, desc: "Number of days to define whether an artifact is old or not. Only affects with analyze commands."
-
       class << self
         def exit_on_failure?
           true

data/lib/mihari/commands/search.rb

@@ -13,12 +13,21 @@ module Mihari
             rule = load_rule(rule)
 
             # validate rule schema
-            validate_rule rule
+            rule = validate_rule(rule)
 
-            analyzer = build_rule_analyzer(**rule)
+            analyzer = build_rule_analyzer(
+              title: rule[:title],
+              description: rule[:description],
+              queries: rule[:queries],
+              tags: rule[:tags],
+              allowed_data_types: rule[:allowed_data_types],
+              disallowed_data_values: rule[:disallowed_data_values],
+              source: rule[:source],
+              id: rule[:id]
+            )
 
-            ignore_old_artifacts = options["ignore_old_artifacts"] || false
-            ignore_threshold = options["ignore_threshold"] || 0
+            ignore_old_artifacts = rule[:ignore_old_artifacts]
+            ignore_threshold = rule[:ignore_threshold]
 
             with_error_handling do
               run_rule_analyzer analyzer, ignore_old_artifacts: ignore_old_artifacts, ignore_threshold: ignore_threshold
@@ -37,13 +46,15 @@ module Mihari
       # @param [Array<Hash>] queries
       # @param [Array<String>, nil] tags
       # @param [Array<String>, nil] allowed_data_types
+      # @param [Array<String>, nil] disallowed_data_values
       # @param [String, nil] source
       #
       # @return [Mihari::Analyzers::Rule]
       #
-      def build_rule_analyzer(title:, description:, queries:, tags: nil, allowed_data_types: nil, source: nil)
+      def build_rule_analyzer(title:, description:, queries:, tags: nil, allowed_data_types: nil, disallowed_data_values: nil, source: nil, id: nil)
         tags = [] if tags.nil?
         allowed_data_types = ALLOWED_DATA_TYPES if allowed_data_types.nil?
+        disallowed_data_values = [] if disallowed_data_values.nil?
 
         Analyzers::Rule.new(
           title: title,
@@ -51,7 +62,9 @@ module Mihari
           tags: tags,
           queries: queries,
           allowed_data_types: allowed_data_types,
-          source: source
+          disallowed_data_values: disallowed_data_values,
+          source: source,
+          id: id
         )
       end
 
@@ -59,8 +72,6 @@ module Mihari
       # Run rule analyzer
       #
       # @param [Mihari::Analyzer::Rule] analyzer
-      # @param [Boolean] ignore_old_artifacts
-      # @param [Integer] ignore_threshold
       #
       # @return [nil]
       #

data/lib/mihari/commands/web.rb

@@ -13,6 +13,10 @@ module Mihari
             host = options["host"] || "localhost"
 
             load_configuration
+
+            # set rack env as production
+            ENV["RACK_ENV"] ||= "production"
+
             Mihari::App.run!(port: port, host: host)
           end
         end

data/lib/mihari/mixins/disallowed_data_value.rb

@@ -0,0 +1,42 @@
+require "mem"
+
+module Mihari
+  module Mixins
+    module DisallowedDataValue
+      include Mem
+
+      #
+      # Normalize a value as a disallowed data value
+      #
+      # @param [String] value Data value
+      #
+      # @return [String, Regexp] Normalized value
+      #
+      def normalize_disallowed_data_value(value)
+        return value if !value.start_with?("/") || !value.end_with?("/")
+
+        # if a value is surrounded by slashes, take it as a regexp
+        value_without_slashes = value[1..-2]
+        Regexp.compile value_without_slashes
+      end
+
+      memoize :normalize_disallowed_data_value
+
+      #
+      # Check whetehr a value is valid format as a disallowed data value
+      #
+      # @param [String] value Data value
+      #
+      # @return [Boolean] true if it is valid, otherwise false
+      #
+      def valid_disallowed_data_value?(value)
+        begin
+          normalize_disallowed_data_value value
+        rescue RegexpError
+          return false
+        end
+        true
+      end
+    end
+  end
+end

data/lib/mihari/mixins/rule.rb

@@ -20,10 +20,12 @@ module Mihari
       end
 
       #
-      # Validate rule schema
+      # Validate rule schema and return a normalized rule
       #
       # @param [Hash] rule
       #
+      # @return [Hash]
+      #
       def validate_rule(rule)
         error_message = "Failed to parse the input as a rule!"
 
@@ -42,6 +44,8 @@ module Mihari
           puts error_message.colorize(:red)
           raise ArgumentError, "Invalid rule schema"
         end
+
+        result.to_h
       end
 
       #

data/lib/mihari/models/artifact.rb

@@ -39,7 +39,7 @@ module Mihari
 
       return false unless ignore_old_artifacts
 
-      days_before = (-ignore_threshold).days.from_now
+      days_before = (-ignore_threshold).days.from_now.utc
       # if an artifact is created before {ignore_threshold} days, ignore it
       #                           within {ignore_threshold} days, do not ignore it
       artifact.created_at < days_before

data/lib/mihari/schemas/analyzer.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require "dry/schema"
+require "dry/validation"
+
+require "mihari/schemas/macros"
+
+module Mihari
+  module Schemas
+    AnalyzerRun = Dry::Schema.Params do
+      required(:title).value(:string)
+      required(:description).value(:string)
+      required(:source).value(:string)
+      required(:artifacts).value(array[:string])
+
+      optional(:tags).value(array[:string]).default([])
+      optional(:ignoreOldArtifacts).value(:bool).default(false)
+      optional(:ignoreThreshold).value(:integer).default(0)
+    end
+
+    class AnalyzerRunContract < Dry::Validation::Contract
+      params(AnalyzerRun)
+    end
+  end
+end

data/lib/mihari/schemas/rule.rb

@@ -63,10 +63,24 @@ module Mihari
       required(:queries).value(:array).each { Analyzer | Spyse | ZoomEye | Urlscan | Crtsh }
 
       optional(:allowed_data_types).value(array[DataTypes]).default(ALLOWED_DATA_TYPES)
+      optional(:disallowed_data_values).value(array[:string]).default([])
+
+      optional(:ignore_old_artifacts).value(:bool).default(false)
+      optional(:ignore_threshold).value(:integer).default(0)
     end
 
     class RuleContract < Dry::Validation::Contract
+      include Mihari::Mixins::DisallowedDataValue
+
       params(Rule)
+
+      rule(:disallowed_data_values) do
+        value.each do |v|
+          unless valid_disallowed_data_value?(v)
+            key.failure("#{v} is not a valid format.")
+          end
+        end
+      end
     end
   end
 end

data/lib/mihari/templates/rule.yml.erb

@@ -2,7 +2,7 @@ title: ... # String (required)
 description: ... # String (required)
 
 id: ... # String (optional)
-author: .. # String (optional)
+author: ... # String (optional)
 created_on: <%= Date.today %> # Date (optional)
 updated_on: <%= Date.today %> # Date (optional)
 
@@ -13,6 +13,10 @@ allowed_data_types: # Array<String> (Optional, defaults to ["hash", "ip", "domai
   - domain
   - url
   - mail
+disallowed_data_values: [] # Array<String> (Optional, defaults to [])
+
+ignore_old_artifacts: true # Whether to ignore old artifacts from checking or not (Optional, defaults to true)
+ignore_threshold: 0 # Number of days to define whether an artifact is old or not (Optional, defaults to 0)
 
 queries: # Array<Hash> (required)
   - analyzer: shodan # String (required)

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "3.0.1"
+  VERSION = "3.4.0"
 end

data/lib/mihari/web/app.rb

@@ -10,6 +10,7 @@ require "mihari/web/helpers/json"
 require "mihari/web/controllers/base_controller"
 
 require "mihari/web/controllers/alerts_controller"
+require "mihari/web/controllers/analyzers_controller"
 require "mihari/web/controllers/artifacts_controller"
 require "mihari/web/controllers/command_controller"
 require "mihari/web/controllers/config_controller"
@@ -26,6 +27,7 @@ module Mihari
     end
 
     use Mihari::Controllers::AlertsController
+    use Mihari::Controllers::AnalyzersController
     use Mihari::Controllers::ArtifactsController
     use Mihari::Controllers::CommandController
     use Mihari::Controllers::ConfigController

data/lib/mihari/web/controllers/analyzers_controller.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Controllers
+    class AnalyzersController < BaseController
+      post "/api/analyzer" do
+        contract = Mihari::Schemas::AnalyzerRunContract.new
+        result = contract.call(params)
+
+        unless result.errors.empty?
+          status 400
+
+          return json(result.errors.to_h)
+        end
+
+        args = result.to_h
+
+        ignore_old_artifacts = args[:ignoreOldArtifacts]
+        ignore_threshold = args[:ignoreThreshold]
+
+        analyzer = Mihari::Analyzers::Basic.new(
+          title: args[:title],
+          description: args[:description],
+          source: args[:source],
+          artifacts: args[:artifacts],
+          tags: args[:tags]
+        )
+        analyzer.ignore_old_artifacts = ignore_old_artifacts
+        analyzer.ignore_threshold = ignore_threshold
+
+        analyzer.run
+
+        status 201
+        body ""
+      end
+    end
+  end
+end