mihari 3.2.0 → 3.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eeee7ec511b59cc1e6d07f47df8b2edc29860dbeeeafebc79ba876efe17f2954
-  data.tar.gz: e3c4fe0b49b20efa5ebdafaecdadbfb35135cbbdf3ff41dd36c68d2eaae644f8
+  metadata.gz: a2fe0203f89908abbc21df8d595b7167a561c826b933166e531298b83f67e085
+  data.tar.gz: 03a455ebd71f5d3c228041351b6ee8a6a2d89b74a7fc2f0d7609293bc82da7d6
 SHA512:
-  metadata.gz: eff2de53ad20576849a81a5ebe96155c6b8747ab4b1775b26816ae6eff7eeb8324f2ad360755e867741a1b6cc778f1dec63d80bbb5439b9e8894536988971183
-  data.tar.gz: 2b1effec63ad4119f7cb521db866e0e9deb2dca7971036530d4cb16ea4a8669d591981ad5517784da13b98b7a3d3aa732538d040b53590b371d8f51acde849e0
+  metadata.gz: aa4a28142eb109d46109d960c1de935ac4632136bc7696b405bfda3d691de2658764e7f796a8e9b1715973e0fc3cc9887997f3ab0dd054e4caa6baf6da40cb8e
+  data.tar.gz: 057c5df7efd59ebf285b4a8d6a17a8857d8fa5df7b0c61f4ddbaf67d3688aaa4c34187af88acfe151a27a21b1d33b5e659347968ba6881e98e3b81114d113eb9

data/README.md

@@ -14,11 +14,12 @@ Mihari is a framework for continuous OSINT based threat hunting.
 
 ## How it works
 
-![img](https://github.com/ninoseki/mihari/raw/master/images/overview.png)
+![img](https://github.com/ninoseki/mihari/raw/master/images/overview.jpg)
 
 - Mihari makes a query against Shodan, Censys, VirusTotal, SecurityTrails, etc. and extracts artifacts (IP addresses, domains, URLs or hashes).
-- Mihari checks whether a DB (SQLite3, PostgreSQL or MySQL) contains the artifacts or not.
+- Mihari checks whether the database (SQLite3, PostgreSQL or MySQL) contains the artifacts or not.
   - If it doesn't contain the artifacts:
+    - Mihari saves artifacts in the database.
     - Mihari creates an alert on TheHive.
     - Mihari sends a notification to Slack.
     - Mihari creates an event on MISP.
@@ -52,6 +53,10 @@ Mihari supports the following services by default.
 
 - [Mihari Knowledge Base](https://www.notion.so/Mihari-Knowledge-Base-266994ff61204428ba6cfcebe40b0bd1)
 
+## Presentations
+
+- [Adversary Infrastructure Tracking with Mihari](https://ninoseki.github.io/presentations/Adversary%20Infrastructure%20Tracking%20with%20Mihari.pdf)
+
 ## License
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/config.ru

@@ -0,0 +1,6 @@
+require "./lib/mihari"
+
+# set rack env as development
+ENV["RACK_ENV"] ||= "development"
+
+run Mihari::App

data/lib/mihari.rb

@@ -9,6 +9,7 @@ require "yaml"
 # Mixins
 require "mihari/mixins/configurable"
 require "mihari/mixins/configuration"
+require "mihari/mixins/disallowed_data_value"
 require "mihari/mixins/hash"
 require "mihari/mixins/refang"
 require "mihari/mixins/retriable"
@@ -30,6 +31,7 @@ module Mihari
   setting :censys_secret, ENV["CENSYS_SECRET"]
   setting :circl_passive_password, ENV["CIRCL_PASSIVE_PASSWORD"]
   setting :circl_passive_username, ENV["CIRCL_PASSIVE_USERNAME"]
+  setting :ipinfo_api_key, ENV["ipinfo_api_key"]
   setting :misp_api_endpoint, ENV["MISP_API_ENDPOINT"]
   setting :misp_api_key, ENV["MISP_API_KEY"]
   setting :onyphe_api_key, ENV["ONYPHE_API_KEY"]

data/lib/mihari/analyzers/rule.rb

@@ -5,6 +5,8 @@ require "uuidtools"
 module Mihari
   module Analyzers
     class Rule < Base
+      include Mihari::Mixins::DisallowedDataValue
+
       option :title
       option :description
       option :queries
@@ -12,6 +14,7 @@ module Mihari
       option :id, default: proc {}
       option :tags, default: proc { [] }
       option :allowed_data_types, default: proc { ALLOWED_DATA_TYPES }
+      option :disallowed_data_values, default: proc { [] }
 
       attr_reader :source
 
@@ -68,12 +71,36 @@ module Mihari
       # - Uniquefy artifacts by #uniq(&:data)
       # - Reject an invalid artifact (for just in case)
       # - Select artifacts with allowed data types
+      # - Reject artifacts with disallowed data values
       #
       # @return [Array<Mihari::Artifact>]
       #
       def normalized_artifacts
         @normalized_artifacts ||= artifacts.uniq(&:data).select(&:valid?).select do |artifact|
           allowed_data_types.include? artifact.data_type
+        end.reject do |artifact|
+          disallowed_data_value? artifact.data
+        end
+      end
+
+      #
+      # Normalized disallowed data values
+      #
+      # @return [Array<Regexp, String>]
+      #
+      def normalized_disallowed_data_values
+        @normalized_disallowed_data_values ||= disallowed_data_values.map { |v| normalize_disallowed_data_value v }
+      end
+
+      #
+      # Check whether a value is a disallowed data value or not
+      #
+      # @return [Boolean]
+      #
+      def disallowed_data_value?(value)
+        normalized_disallowed_data_values.any? do |disallowed_data_value|
+          return value == disallowed_data_value if disallowed_data_value.is_a?(String)
+          return disallowed_data_value.match?(value) if disallowed_data_value.is_a?(Regexp)
         end
       end
 

data/lib/mihari/cli/analyzer.rb

@@ -22,6 +22,10 @@ require "mihari/commands/json"
 module Mihari
   module CLI
     class Analyzer < Base
+      class_option :ignore_old_artifacts, type: :boolean, default: false, desc: "Whether to ignore old artifacts from checking or not."
+      class_option :ignore_threshold, type: :numeric, default: 0, desc: "Number of days to define whether an artifact is old or not."
+      class_option :config, type: :string, desc: "Path to the config file"
+
       include Mihari::Commands::BinaryEdge
       include Mihari::Commands::Censys
       include Mihari::Commands::CIRCL

data/lib/mihari/cli/base.rb

@@ -12,11 +12,6 @@ module Mihari
       include Mihari::Mixins::Hash
       include Mixins::Utils
 
-      class_option :config, type: :string, desc: "Path to the config file"
-
-      class_option :ignore_old_artifacts, type: :boolean, default: false, desc: "Whether to ignore old artifacts from checking or not. Only affects with analyze commands."
-      class_option :ignore_threshold, type: :numeric, default: 0, desc: "Number of days to define whether an artifact is old or not. Only affects with analyze commands."
-
       class << self
         def exit_on_failure?
           true

data/lib/mihari/commands/init.rb

@@ -5,10 +5,10 @@ require "colorize"
 module Mihari
   module Commands
     module Initialization
-      def self.included(thor)
-        include Mixins::Configuration
-        include Mixins::Rule
+      include Mixins::Configuration
+      include Mixins::Rule
 
+      def self.included(thor)
         thor.class_eval do
           desc "config", "Create a config file"
           method_option :filename, type: :string, default: "mihari.yml"
@@ -37,7 +37,7 @@ module Mihari
 
             initialize_rule_yaml filename
 
-            puts "The rule file is created as #{filename}.".colorize(:blue)
+            puts "The rule file is initialized as #{filename}.".colorize(:blue)
           end
         end
       end

data/lib/mihari/commands/search.rb

@@ -8,17 +8,27 @@ module Mihari
       def self.included(thor)
         thor.class_eval do
           desc "search [RULE]", "Search by a rule"
+          method_option :config, type: :string, desc: "Path to the config file"
           def search_by_rule(rule)
             # convert str(YAML) to hash or str(path/YAML file) to hash
             rule = load_rule(rule)
 
             # validate rule schema
-            validate_rule rule
+            rule = validate_rule(rule)
 
-            analyzer = build_rule_analyzer(**rule)
+            analyzer = build_rule_analyzer(
+              title: rule[:title],
+              description: rule[:description],
+              queries: rule[:queries],
+              tags: rule[:tags],
+              allowed_data_types: rule[:allowed_data_types],
+              disallowed_data_values: rule[:disallowed_data_values],
+              source: rule[:source],
+              id: rule[:id]
+            )
 
-            ignore_old_artifacts = options["ignore_old_artifacts"] || false
-            ignore_threshold = options["ignore_threshold"] || 0
+            ignore_old_artifacts = rule[:ignore_old_artifacts]
+            ignore_threshold = rule[:ignore_threshold]
 
             with_error_handling do
               run_rule_analyzer analyzer, ignore_old_artifacts: ignore_old_artifacts, ignore_threshold: ignore_threshold
@@ -37,13 +47,15 @@ module Mihari
       # @param [Array<Hash>] queries
       # @param [Array<String>, nil] tags
       # @param [Array<String>, nil] allowed_data_types
+      # @param [Array<String>, nil] disallowed_data_values
       # @param [String, nil] source
       #
       # @return [Mihari::Analyzers::Rule]
       #
-      def build_rule_analyzer(title:, description:, queries:, tags: nil, allowed_data_types: nil, source: nil)
+      def build_rule_analyzer(title:, description:, queries:, tags: nil, allowed_data_types: nil, disallowed_data_values: nil, source: nil, id: nil)
         tags = [] if tags.nil?
         allowed_data_types = ALLOWED_DATA_TYPES if allowed_data_types.nil?
+        disallowed_data_values = [] if disallowed_data_values.nil?
 
         Analyzers::Rule.new(
           title: title,
@@ -51,7 +63,9 @@ module Mihari
           tags: tags,
           queries: queries,
           allowed_data_types: allowed_data_types,
-          source: source
+          disallowed_data_values: disallowed_data_values,
+          source: source,
+          id: id
         )
       end
 
@@ -59,8 +73,6 @@ module Mihari
       # Run rule analyzer
       #
       # @param [Mihari::Analyzer::Rule] analyzer
-      # @param [Boolean] ignore_old_artifacts
-      # @param [Integer] ignore_threshold
       #
       # @return [nil]
       #

data/lib/mihari/commands/web.rb

@@ -8,11 +8,16 @@ module Mihari
           desc "web", "Launch the web app"
           method_option :port, type: :numeric, default: 9292
           method_option :host, type: :string, default: "localhost"
+          method_option :config, type: :string, desc: "Path to the config file"
           def web
             port = options["port"].to_i || 9292
             host = options["host"] || "localhost"
 
             load_configuration
+
+            # set rack env as production
+            ENV["RACK_ENV"] ||= "production"
+
             Mihari::App.run!(port: port, host: host)
           end
         end

data/lib/mihari/mixins/disallowed_data_value.rb

@@ -0,0 +1,42 @@
+require "mem"
+
+module Mihari
+  module Mixins
+    module DisallowedDataValue
+      include Mem
+
+      #
+      # Normalize a value as a disallowed data value
+      #
+      # @param [String] value Data value
+      #
+      # @return [String, Regexp] Normalized value
+      #
+      def normalize_disallowed_data_value(value)
+        return value if !value.start_with?("/") || !value.end_with?("/")
+
+        # if a value is surrounded by slashes, take it as a regexp
+        value_without_slashes = value[1..-2]
+        Regexp.compile value_without_slashes
+      end
+
+      memoize :normalize_disallowed_data_value
+
+      #
+      # Check whetehr a value is valid format as a disallowed data value
+      #
+      # @param [String] value Data value
+      #
+      # @return [Boolean] true if it is valid, otherwise false
+      #
+      def valid_disallowed_data_value?(value)
+        begin
+          normalize_disallowed_data_value value
+        rescue RegexpError
+          return false
+        end
+        true
+      end
+    end
+  end
+end

data/lib/mihari/mixins/rule.rb

@@ -20,10 +20,12 @@ module Mihari
       end
 
       #
-      # Validate rule schema
+      # Validate rule schema and return a normalized rule
       #
       # @param [Hash] rule
       #
+      # @return [Hash]
+      #
       def validate_rule(rule)
         error_message = "Failed to parse the input as a rule!"
 
@@ -42,6 +44,8 @@ module Mihari
           puts error_message.colorize(:red)
           raise ArgumentError, "Invalid rule schema"
         end
+
+        result.to_h
       end
 
       #

data/lib/mihari/models/alert.rb

@@ -18,8 +18,8 @@ module Mihari
       # @param [String, nil] source
       # @param [String, nil] tag_name
       # @param [String, nil] title
-      # @param [String, nil] from_at
-      # @param [String, nil] to_at
+      # @param [DateTime, nil] from_at
+      # @param [DateTime, nil] to_at
       # @param [Integer, nil] limit
       # @param [Integer, nil] page
       #
@@ -34,7 +34,15 @@ module Mihari
 
         offset = (page - 1) * limit
 
-        relation = build_relation(artifact_data: artifact_data, title: title, description: description, source: source, tag_name: tag_name, from_at: from_at, to_at: to_at)
+        relation = build_relation(
+          artifact_data: artifact_data,
+          title: title,
+          description: description,
+          source: source,
+          tag_name: tag_name,
+          from_at: from_at,
+          to_at: to_at
+        )
 
         alerts = relation.limit(limit).offset(offset).order(id: :desc)
 
@@ -54,13 +62,21 @@ module Mihari
       # @param [String, nil] source
       # @param [String, nil] tag_name
       # @param [String, nil] title
-      # @param [String, nil] from_at
-      # @param [String, nil] to_at
+      # @param [DateTime, nil] from_at
+      # @param [DateTime, nil] to_at
       #
       # @return [Integer]
       #
       def count(artifact_data: nil, description: nil, source: nil, tag_name: nil, title: nil, from_at: nil, to_at: nil)
-        relation = build_relation(artifact_data: artifact_data, title: title, description: description, source: source, tag_name: tag_name, from_at: from_at, to_at: to_at)
+        relation = build_relation(
+          artifact_data: artifact_data,
+          title: title,
+          description: description,
+          source: source,
+          tag_name: tag_name,
+          from_at: from_at,
+          to_at: to_at
+        )
         relation.distinct("alerts.id").count
       end
 
@@ -68,11 +84,9 @@ module Mihari
 
       def build_relation(artifact_data: nil, title: nil, description: nil, source: nil, tag_name: nil, from_at: nil, to_at: nil)
         relation = self
-        relation = joins(:tags) if tag_name
-        relation = joins(:artifacts) if artifact_data
 
-        relation = relation.where(artifacts: { data: artifact_data }) if artifact_data
-        relation = relation.where(tags: { name: tag_name }) if tag_name
+        relation = relation.joins(:artifacts).where(artifacts: { data: artifact_data }) if artifact_data
+        relation = relation.joins(:tags).where(tags: { name: tag_name }) if tag_name
 
         relation = relation.where(source: source) if source
         relation = relation.where(title: title) if title

data/lib/mihari/schemas/configuration.rb

@@ -13,6 +13,7 @@ module Mihari
       optional(:censys_secret).value(:string)
       optional(:circl_passive_password).value(:string)
       optional(:circl_passive_username).value(:string)
+      optional(:ipinfo_api_key).value(:string)
       optional(:misp_api_endpoint).value(:string)
       optional(:misp_api_key).value(:string)
       optional(:onyphe_api_key).value(:string)

data/lib/mihari/schemas/rule.rb

@@ -63,10 +63,24 @@ module Mihari
       required(:queries).value(:array).each { Analyzer | Spyse | ZoomEye | Urlscan | Crtsh }
 
       optional(:allowed_data_types).value(array[DataTypes]).default(ALLOWED_DATA_TYPES)
+      optional(:disallowed_data_values).value(array[:string]).default([])
+
+      optional(:ignore_old_artifacts).value(:bool).default(false)
+      optional(:ignore_threshold).value(:integer).default(0)
     end
 
     class RuleContract < Dry::Validation::Contract
+      include Mihari::Mixins::DisallowedDataValue
+
       params(Rule)
+
+      rule(:disallowed_data_values) do
+        value.each do |v|
+          unless valid_disallowed_data_value?(v)
+            key.failure("#{v} is not a valid format.")
+          end
+        end
+      end
     end
   end
 end

data/lib/mihari/templates/rule.yml.erb

@@ -2,7 +2,7 @@ title: ... # String (required)
 description: ... # String (required)
 
 id: ... # String (optional)
-author: .. # String (optional)
+author: ... # String (optional)
 created_on: <%= Date.today %> # Date (optional)
 updated_on: <%= Date.today %> # Date (optional)
 
@@ -13,6 +13,10 @@ allowed_data_types: # Array<String> (Optional, defaults to ["hash", "ip", "domai
   - domain
   - url
   - mail
+disallowed_data_values: [] # Array<String> (Optional, defaults to [])
+
+ignore_old_artifacts: true # Whether to ignore old artifacts from checking or not (Optional, defaults to true)
+ignore_threshold: 0 # Number of days to define whether an artifact is old or not (Optional, defaults to 0)
 
 queries: # Array<Hash> (required)
   - analyzer: shodan # String (required)

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "3.2.0"
+  VERSION = "3.5.0"
 end