mihari 2.2.0 → 3.0.0

data/lib/mihari/mixins/refang.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Mixins
+    module Refang
+      #
+      # Refang defanged indicator
+      #
+      # @param [String] indicator
+      #
+      # @return [String]
+      #
+      def refang(indicator)
+        return indicator.gsub("[.]", ".").gsub("(.)", ".") if indicator.is_a?(String)
+
+        # for RSpec & Ruby 2.7
+        indicator
+      end
+    end
+  end
+end

data/lib/mihari/mixins/retriable.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Mixins
+    module Retriable
+      #
+      # Retry on error
+      #
+      # @param [Integer] times
+      # @param [Integer] interval
+      #
+      # @return [nil]
+      #
+      def retry_on_error(times: 3, interval: 10)
+        try = 0
+        begin
+          try += 1
+          yield
+        rescue Errno::ECONNRESET, Errno::ECONNABORTED, Errno::EPIPE, OpenSSL::SSL::SSLError, Timeout::Error, RetryableError => e
+          sleep interval
+          retry if try < times
+          raise e
+        end
+      end
+    end
+  end
+end

data/lib/mihari/mixins/rule.rb

@@ -0,0 +1,79 @@
+require "date"
+require "pathname"
+require "yaml"
+require "erb"
+
+module Mihari
+  module Mixins
+    module Rule
+      #
+      # Load rule into hash
+      #
+      # @param [String] path Path to YAML file or YAML string
+      #
+      # @return [Hash]
+      #
+      def load_rule(path)
+        return YAML.safe_load(File.read(path), permitted_classes: [Date], symbolize_names: true) if Pathname(path).exist?
+
+        YAML.safe_load(path, symbolize_names: true)
+      end
+
+      #
+      # Validate rule schema
+      #
+      # @param [Hash] rule
+      #
+      def validate_rule(rule)
+        error_message = "Failed to parse the input as a rule!"
+
+        contract = Schemas::RuleContract.new
+        begin
+          result = contract.call(rule)
+          unless result.errors.empty?
+            messages = result.errors.messages.map do |message|
+              path = message.path.map(&:to_s).join
+              "#{path} #{message.text}"
+            end
+            puts error_message.colorize(:red)
+            raise ArgumentError, messages.join("\n")
+          end
+        rescue NoMethodError
+          puts error_message.colorize(:red)
+          raise ArgumentError, "Invalid rule schema"
+        end
+      end
+
+      #
+      # Returns a template for rule
+      #
+      # @return [String] A template for rule
+      #
+      def rule_template
+        # Use ERB to fill created_on and updated_on with Date.today
+        data = File.read(File.expand_path("../templates/rule.yml.erb", __dir__))
+        template = ERB.new(data)
+        data = template.result
+
+        # validate the template of rule for just in case
+        rule = YAML.safe_load(data, permitted_classes: [Date], symbolize_names: true)
+        validate_rule rule
+
+        data
+      end
+
+      #
+      # Create (blank) rule file
+      #
+      # @param [String] filename
+      # @param [Dry::Files] files
+      # @param [String] template
+      #
+      # @return [nil]
+      #
+      def initialize_rule_yaml(filename, files = Dry::Files.new, template: rule_template)
+        files.write(filename, template)
+      end
+    end
+  end
+end

data/lib/mihari/models/alert.rb

@@ -10,6 +10,21 @@ module Mihari
     has_many :tags, through: :taggings
 
     class << self
+      #
+      # Search alerts
+      #
+      # @param [String, nil] artifact_data
+      # @param [String, nil] description
+      # @param [String, nil] source
+      # @param [String, nil] tag_name
+      # @param [String, nil] title
+      # @param [String, nil] from_at
+      # @param [String, nil] to_at
+      # @param [Integer, nil] limit
+      # @param [Integer, nil] page
+      #
+      # @return [Array<Hash>]
+      #
       def search(artifact_data: nil, description: nil, source: nil, tag_name: nil, title: nil, from_at: nil, to_at: nil, limit: 10, page: 1)
         limit = limit.to_i
         raise ArgumentError, "limit should be bigger than zero" unless limit.positive?
@@ -20,7 +35,6 @@ module Mihari
         offset = (page - 1) * limit
 
         relation = build_relation(artifact_data: artifact_data, title: title, description: description, source: source, tag_name: tag_name, from_at: from_at, to_at: to_at)
-        # relation = relation.group("alerts.id")
 
         alerts = relation.limit(limit).offset(offset).order(id: :desc)
 
@@ -32,6 +46,19 @@ module Mihari
         end
       end
 
+      #
+      # Count alerts
+      #
+      # @param [String, nil] artifact_data
+      # @param [String, nil] description
+      # @param [String, nil] source
+      # @param [String, nil] tag_name
+      # @param [String, nil] title
+      # @param [String, nil] from_at
+      # @param [String, nil] to_at
+      #
+      # @return [Integer]
+      #
       def count(artifact_data: nil, description: nil, source: nil, tag_name: nil, title: nil, from_at: nil, to_at: nil)
         relation = build_relation(artifact_data: artifact_data, title: title, description: description, source: source, tag_name: tag_name, from_at: from_at, to_at: to_at)
         relation.distinct("alerts.id").count

data/lib/mihari/models/artifact.rb

@@ -16,13 +16,23 @@ end
 module Mihari
   class Artifact < ActiveRecord::Base
     include ActiveModel::Validations
+
     validates_with ArtifactValidator
 
     def initialize(attributes)
       super
+
       self.data_type = TypeChecker.type(data)
     end
 
+    #
+    # Check uniqueness of artifact
+    #
+    # @param [Boolean] ignore_old_artifacts
+    # @param [Integer] ignore_threshold
+    #
+    # @return [Boolean] true if it is unique. Otherwise false.
+    #
     def unique?(ignore_old_artifacts: false, ignore_threshold: 0)
       artifact = self.class.where(data: data).order(created_at: :desc).first
       return true if artifact.nil?

data/lib/mihari/notifiers/base.rb

@@ -3,11 +3,19 @@
 module Mihari
   module Notifiers
     class Base
-      # @return [true, false]
+      # Validate notifier availability
+      #
+      # @return [Boolean]
+      #
       def valid?
         raise NotImplementedError, "You must implement #{self.class}##{__method__}"
       end
 
+      #
+      # Send a notification
+      #
+      # @return [nil]
+      #
       def notify
         raise NotImplementedError, "You must implement #{self.class}##{__method__}"
       end

data/lib/mihari/notifiers/exception_notifier.rb

@@ -24,10 +24,25 @@ module Mihari
         notify_to_slack(text: clean_message, attachments: attachments) if @slack.valid?
       end
 
+      #
+      # Send notification to Slack
+      #
+      # @param [String] text
+      # @param [Array<Hash>] attachments
+      #
+      # @return [nil]
+      #
       def notify_to_slack(text:, attachments:)
         @slack.notify(text: text, attachments: attachments)
       end
 
+      #
+      # Send notification to STDOUT
+      #
+      # @param [Exception] exception
+      #
+      # @return [nil]
+      #
       def notify_to_stdout(exception)
         text = to_text(exception.class).chomp
         message = "#{text}: #{exception.message}"
@@ -35,6 +50,14 @@ module Mihari
         puts format_backtrace(exception.backtrace) if exception.backtrace
       end
 
+      #
+      # Convert exception to attachments (for Slack)
+      #
+      # @param [Exception] exception
+      # @param [String] clean_message
+      #
+      # @return [Array<Hash>]
+      #
       def to_attachments(exception, clean_message)
         text = to_text(exception.class)
         backtrace = exception.backtrace
@@ -43,12 +66,27 @@ module Mihari
         [color: @color, text: text, fields: fields, mrkdwn_in: %w[text fields]]
       end
 
+      #
+      # Convert exception class to text
+      #
+      # @param [Class<Exception>] exception_class
+      #
+      # @return [String]
+      #
       def to_text(exception_class)
         measure_word = /^[aeiou]/i.match?(exception_class.to_s) ? "An" : "A"
         exception_name = "*#{measure_word}* `#{exception_class}`"
         "#{exception_name} *occured in background*\n"
       end
 
+      #
+      # Convert clean_message and backtrace into fields (for Slack)
+      #
+      # @param [String] clean_message
+      # @param [Array] backtrace
+      #
+      # @return [Array<Hash>]
+      #
       def to_fields(clean_message, backtrace)
         fields = [
           { title: "Exception", value: clean_message },
@@ -62,12 +100,24 @@ module Mihari
         fields
       end
 
+      #
+      # Hostname of runnning instance
+      #
+      # @return [String]
+      #
       def hostname
         Socket.gethostname
       rescue StandardError => _e
         "N/A"
       end
 
+      #
+      # Format backtrace in string
+      #
+      # @param [Array] backtrace
+      #
+      # @return [String]
+      #
       def format_backtrace(backtrace)
         return nil unless backtrace
 

data/lib/mihari/notifiers/slack.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "slack-notifier"
-require "mihari/slack_monkeypatch"
 
 module Mihari
   module Notifiers
@@ -10,22 +9,51 @@ module Mihari
       SLACK_CHANNEL_KEY = "SLACK_CHANNEL"
       DEFAULT_USERNAME = "mihari"
 
+      #
+      # Slack channel to post
+      #
+      # @return [String]
+      #
       def slack_channel
         Mihari.config.slack_channel || "#general"
       end
 
+      #
+      # Slack webhook URL
+      #
+      # @return [String]
+      #
       def slack_webhook_url
         Mihari.config.slack_webhook_url
       end
 
+      #
+      # Check Slack webhook URL is set
+      #
+      # @return [Boolean]
+      #
       def slack_webhook_url?
         !Mihari.config.slack_webhook_url.nil?
       end
 
+      #
+      # Check Slack webhook URL is set. Alias of #slack_webhook_url?.
+      #
+      # @return [Boolean]
+      #
       def valid?
         slack_webhook_url?
       end
 
+      #
+      # Send notification to Slack
+      #
+      # @param [String] text
+      # @param [Array<Hash>] attachments
+      # @param [Boolean] mrkdwn
+      #
+      # @return [nil]
+      #
       def notify(text:, attachments: [], mrkdwn: true)
         notifier = ::Slack::Notifier.new(slack_webhook_url, channel: slack_channel, username: DEFAULT_USERNAME)
         notifier.post(text: text, attachments: attachments, mrkdwn: mrkdwn)

data/lib/mihari/schemas/configuration.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require "dry/schema"
+require "dry/validation"
+
+require "mihari/schemas/macros"
+
+module Mihari
+  module Schemas
+    Configuration = Dry::Schema.Params do
+      optional(:binaryedge_api_key).value(:string)
+      optional(:censys_id).value(:string)
+      optional(:censys_secret).value(:string)
+      optional(:circl_passive_password).value(:string)
+      optional(:circl_passive_username).value(:string)
+      optional(:misp_api_endpoint).value(:string)
+      optional(:misp_api_key).value(:string)
+      optional(:onyphe_api_key).value(:string)
+      optional(:otx_api_key).value(:string)
+      optional(:passivetotal_api_key).value(:string)
+      optional(:passivetotal_username).value(:string)
+      optional(:pulsedive_api_key).value(:string)
+      optional(:securitytrails_api_key).value(:string)
+      optional(:shodan_api_key).value(:string)
+      optional(:slack_channel).value(:string)
+      optional(:slack_webhook_url).value(:string)
+      optional(:spyse_api_key).value(:string)
+      optional(:thehive_api_endpoint).value(:string)
+      optional(:thehive_api_key).value(:string)
+      optional(:urlscan_api_key).value(:string)
+      optional(:virustotal_api_key).value(:string)
+      optional(:zoomeye_api_key).value(:string)
+      optional(:webhook_url).value(:string)
+      optional(:webhook_use_json_body).value(:bool)
+      optional(:database).value(:string)
+    end
+
+    class ConfigurationContract < Dry::Validation::Contract
+      params(Configuration)
+    end
+  end
+end

data/lib/mihari/schemas/macros.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require "dry/types"
+
+module Dry
+  module Schema
+    module Macros
+      class DSL
+        def default(value)
+          schema_dsl.before(:rule_applier) do |result|
+            result.update(name => value) unless result[name]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mihari/schemas/rule.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require "dry/schema"
+require "dry/validation"
+require "dry/types"
+
+require "mihari/schemas/macros"
+
+module Mihari
+  module Types
+    include Dry.Types()
+  end
+
+  DataTypes = Types::String.enum(*ALLOWED_DATA_TYPES)
+
+  AnalyzerTypes = Types::String.enum(
+    "binaryedge", "censys", "circl", "dnpedia", "dnstwister",
+    "onyphe", "otx", "passivetotal", "pulsedive", "securitytrails",
+    "shodan", "virustotal"
+  )
+
+  module Schemas
+    Analyzer = Dry::Schema.Params do
+      required(:analyzer).value(AnalyzerTypes)
+      required(:query).value(:string)
+    end
+
+    Spyse = Dry::Schema.Params do
+      required(:analyzer).value(Types::String.enum("spyse"))
+      required(:query).value(:string)
+      required(:type).value(Types::String.enum("ip", "domain"))
+    end
+
+    ZoomEye = Dry::Schema.Params do
+      required(:analyzer).value(Types::String.enum("zoomeye"))
+      required(:query).value(:string)
+      required(:type).value(Types::String.enum("host", "web"))
+    end
+
+    Crtsh = Dry::Schema.Params do
+      required(:analyzer).value(Types::String.enum("crtsh"))
+      required(:query).value(:string)
+      optional(:exclude_expired).value(:bool).default(true)
+    end
+
+    Urlscan = Dry::Schema.Params do
+      required(:analyzer).value(Types::String.enum("urlscan"))
+      required(:query).value(:string)
+      optional(:use_similarity).value(:bool).default(true)
+    end
+
+    Rule = Dry::Schema.Params do
+      required(:title).value(:string)
+      required(:description).value(:string)
+
+      optional(:tags).value(array[:string]).default([])
+      optional(:id).value(:string)
+
+      optional(:author).value(:string)
+      optional(:created_on).value(:date)
+      optional(:updated_on).value(:date)
+
+      required(:queries).value(:array).each { Analyzer | Spyse | ZoomEye | Urlscan | Crtsh }
+
+      optional(:allowed_data_types).value(array[DataTypes]).default(ALLOWED_DATA_TYPES)
+    end
+
+    class RuleContract < Dry::Validation::Contract
+      params(Rule)
+    end
+  end
+end