mihari 3.2.0 → 3.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eeee7ec511b59cc1e6d07f47df8b2edc29860dbeeeafebc79ba876efe17f2954
-  data.tar.gz: e3c4fe0b49b20efa5ebdafaecdadbfb35135cbbdf3ff41dd36c68d2eaae644f8
+  metadata.gz: 56538b2cc9269fa7e92a2c8cd98746caae7a480b7961fb83482a2189f5318992
+  data.tar.gz: 9c4561c9e820a42b9b4d65b1e0ad02901034cc7eb124d7c5409418cab43bc5bd
 SHA512:
-  metadata.gz: eff2de53ad20576849a81a5ebe96155c6b8747ab4b1775b26816ae6eff7eeb8324f2ad360755e867741a1b6cc778f1dec63d80bbb5439b9e8894536988971183
-  data.tar.gz: 2b1effec63ad4119f7cb521db866e0e9deb2dca7971036530d4cb16ea4a8669d591981ad5517784da13b98b7a3d3aa732538d040b53590b371d8f51acde849e0
+  metadata.gz: 0b43cf2ee5e73607593e2c140dbc4ec70dcc8ad1eb436914a016f852f8dabed7f1aafd2f048565356f3fbed865540488e0766bc7f3a86bd6845c722419472abe
+  data.tar.gz: 485bbcd01bcd8016b3a50b5e61fcac4cef8f527812570c734eff8a0fe6a75fc3b8e79e50d2a147c4edc20eee0370609ec5c2021bd28172bedff484bd04707379

data/README.md

@@ -14,11 +14,12 @@ Mihari is a framework for continuous OSINT based threat hunting.
 
 ## How it works
 
-![img](https://github.com/ninoseki/mihari/raw/master/images/overview.png)
+![img](https://github.com/ninoseki/mihari/raw/master/images/overview.jpg)
 
 - Mihari makes a query against Shodan, Censys, VirusTotal, SecurityTrails, etc. and extracts artifacts (IP addresses, domains, URLs or hashes).
-- Mihari checks whether a DB (SQLite3, PostgreSQL or MySQL) contains the artifacts or not.
+- Mihari checks whether the database (SQLite3, PostgreSQL or MySQL) contains the artifacts or not.
   - If it doesn't contain the artifacts:
+    - Mihari saves artifacts in the database.
     - Mihari creates an alert on TheHive.
     - Mihari sends a notification to Slack.
     - Mihari creates an event on MISP.

data/config.ru

@@ -0,0 +1,6 @@
+require "./lib/mihari"
+
+# set rack env as development
+ENV["RACK_ENV"] ||= "development"
+
+run Mihari::App

data/lib/mihari.rb

@@ -9,6 +9,7 @@ require "yaml"
 # Mixins
 require "mihari/mixins/configurable"
 require "mihari/mixins/configuration"
+require "mihari/mixins/disallowed_data_value"
 require "mihari/mixins/hash"
 require "mihari/mixins/refang"
 require "mihari/mixins/retriable"

data/lib/mihari/analyzers/rule.rb

@@ -5,6 +5,8 @@ require "uuidtools"
 module Mihari
   module Analyzers
     class Rule < Base
+      include Mihari::Mixins::DisallowedDataValue
+
       option :title
       option :description
       option :queries
@@ -12,6 +14,7 @@ module Mihari
       option :id, default: proc {}
       option :tags, default: proc { [] }
       option :allowed_data_types, default: proc { ALLOWED_DATA_TYPES }
+      option :disallowed_data_values, default: proc { [] }
 
       attr_reader :source
 
@@ -68,12 +71,36 @@ module Mihari
       # - Uniquefy artifacts by #uniq(&:data)
       # - Reject an invalid artifact (for just in case)
       # - Select artifacts with allowed data types
+      # - Reject artifacts with disallowed data values
       #
       # @return [Array<Mihari::Artifact>]
       #
       def normalized_artifacts
         @normalized_artifacts ||= artifacts.uniq(&:data).select(&:valid?).select do |artifact|
           allowed_data_types.include? artifact.data_type
+        end.reject do |artifact|
+          disallowed_data_value? artifact.data
+        end
+      end
+
+      #
+      # Normalized disallowed data values
+      #
+      # @return [Array<Regexp, String>]
+      #
+      def normalized_disallowed_data_values
+        @normalized_disallowed_data_values ||= disallowed_data_values.map { |v| normalize_disallowed_data_value v }
+      end
+
+      #
+      # Check whether a value is a disallowed data value or not
+      #
+      # @return [Boolean]
+      #
+      def disallowed_data_value?(value)
+        normalized_disallowed_data_values.any? do |disallowed_data_value|
+          return value == disallowed_data_value if disallowed_data_value.is_a?(String)
+          return disallowed_data_value.match?(value) if disallowed_data_value.is_a?(Regexp)
         end
       end
 

data/lib/mihari/commands/search.rb

@@ -37,13 +37,15 @@ module Mihari
       # @param [Array<Hash>] queries
       # @param [Array<String>, nil] tags
       # @param [Array<String>, nil] allowed_data_types
+      # @param [Array<String>, nil] disallowed_data_values
       # @param [String, nil] source
       #
       # @return [Mihari::Analyzers::Rule]
       #
-      def build_rule_analyzer(title:, description:, queries:, tags: nil, allowed_data_types: nil, source: nil)
+      def build_rule_analyzer(title:, description:, queries:, tags: nil, allowed_data_types: nil, disallowed_data_values: nil, source: nil)
         tags = [] if tags.nil?
         allowed_data_types = ALLOWED_DATA_TYPES if allowed_data_types.nil?
+        disallowed_data_values = [] if disallowed_data_values.nil?
 
         Analyzers::Rule.new(
           title: title,
@@ -51,6 +53,7 @@ module Mihari
           tags: tags,
           queries: queries,
           allowed_data_types: allowed_data_types,
+          disallowed_data_values: disallowed_data_values,
           source: source
         )
       end

data/lib/mihari/commands/web.rb

@@ -13,6 +13,10 @@ module Mihari
             host = options["host"] || "localhost"
 
             load_configuration
+
+            # set rack env as production
+            ENV["RACK_ENV"] ||= "production"
+
             Mihari::App.run!(port: port, host: host)
           end
         end

data/lib/mihari/mixins/disallowed_data_value.rb

@@ -0,0 +1,42 @@
+require "mem"
+
+module Mihari
+  module Mixins
+    module DisallowedDataValue
+      include Mem
+
+      #
+      # Normalize a value as a disallowed data value
+      #
+      # @param [String] value Data value
+      #
+      # @return [String, Regexp] Normalized value
+      #
+      def normalize_disallowed_data_value(value)
+        return value if !value.start_with?("/") || !value.end_with?("/")
+
+        # if a value is surrounded by slashes, take it as a regexp
+        value_without_slashes = value[1..-2]
+        Regexp.compile value_without_slashes
+      end
+
+      memoize :normalize_disallowed_data_value
+
+      #
+      # Check whetehr a value is valid format as a disallowed data value
+      #
+      # @param [String] value Data value
+      #
+      # @return [Boolean] true if it is valid, otherwise false
+      #
+      def valid_disallowed_data_value?(value)
+        begin
+          normalize_disallowed_data_value value
+        rescue RegexpError
+          return false
+        end
+        true
+      end
+    end
+  end
+end

data/lib/mihari/schemas/rule.rb

@@ -63,10 +63,21 @@ module Mihari
       required(:queries).value(:array).each { Analyzer | Spyse | ZoomEye | Urlscan | Crtsh }
 
       optional(:allowed_data_types).value(array[DataTypes]).default(ALLOWED_DATA_TYPES)
+      optional(:disallowed_data_values).value(array[:string]).default([])
     end
 
     class RuleContract < Dry::Validation::Contract
+      include Mihari::Mixins::DisallowedDataValue
+
       params(Rule)
+
+      rule(:disallowed_data_values) do
+        value.each do |v|
+          unless valid_disallowed_data_value?(v)
+            key.failure("#{v} is not a valid format.")
+          end
+        end
+      end
     end
   end
 end

data/lib/mihari/templates/rule.yml.erb

@@ -2,7 +2,7 @@ title: ... # String (required)
 description: ... # String (required)
 
 id: ... # String (optional)
-author: .. # String (optional)
+author: ... # String (optional)
 created_on: <%= Date.today %> # Date (optional)
 updated_on: <%= Date.today %> # Date (optional)
 
@@ -13,6 +13,7 @@ allowed_data_types: # Array<String> (Optional, defaults to ["hash", "ip", "domai
   - domain
   - url
   - mail
+disallowed_data_values: [] # Array<String> (Optional, defaults to [])
 
 queries: # Array<Hash> (required)
   - analyzer: shodan # String (required)

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "3.2.0"
+  VERSION = "3.3.0"
 end

data/lib/mihari/web/app.rb

@@ -16,6 +16,7 @@ require "mihari/web/controllers/command_controller"
 require "mihari/web/controllers/config_controller"
 require "mihari/web/controllers/sources_controller"
 require "mihari/web/controllers/tags_controller"
+
 module Mihari
   class App < Sinatra::Base
     set :root, File.dirname(__FILE__)

data/mihari.gemspec

@@ -68,7 +68,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "passivetotalx", "~> 0.1"
   spec.add_dependency "public_suffix", "~> 4.0"
   spec.add_dependency "pulsedive", "~> 0.1"
-  spec.add_dependency "puma", "~> 5.3"
+  spec.add_dependency "puma", "~> 5.4"
   spec.add_dependency "rack", "~> 2.2"
   spec.add_dependency "rack-contrib", "~> 2.3"
   spec.add_dependency "safe_shell", "~> 1.1"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mihari
 version: !ruby/object:Gem::Version
-  version: 3.2.0
+  version: 3.3.0
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-07-25 00:00:00.000000000 Z
+date: 2021-08-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -604,14 +604,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.3'
+        version: '5.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.3'
+        version: '5.4'
 - !ruby/object:Gem::Dependency
   name: rack
   requirement: !ruby/object:Gem::Requirement
@@ -872,6 +872,7 @@ files:
 - bin/console
 - bin/setup
 - build_frontend.sh
+- config.ru
 - config/pre_commit.yml
 - docker/Dockerfile
 - examples/ipinfo_hosted_domains.rb
@@ -879,7 +880,7 @@ files:
 - images/alert.png
 - images/logo.png
 - images/misp.png
-- images/overview.png
+- images/overview.jpg
 - images/slack.png
 - images/tines.png
 - images/web_alerts.png
@@ -943,6 +944,7 @@ files:
 - lib/mihari/errors.rb
 - lib/mihari/mixins/configurable.rb
 - lib/mihari/mixins/configuration.rb
+- lib/mihari/mixins/disallowed_data_value.rb
 - lib/mihari/mixins/hash.rb
 - lib/mihari/mixins/refang.rb
 - lib/mihari/mixins/retriable.rb