mihari 4.3.0 → 4.4.0

data/lib/mihari/emitters/slack.rb

@@ -109,48 +109,46 @@ module Mihari
     end
 
     class Slack < Base
-      SLACK_WEBHOOK_URL_KEY = "SLACK_WEBHOOK_URL"
-      SLACK_CHANNEL_KEY = "SLACK_CHANNEL"
-      DEFAULT_USERNAME = "mihari"
+      DEFAULT_CHANNEL = "#general"
+      DEFAULT_USERNAME = "Mihari"
+
+      # @return [String, nil]
+      attr_reader :webhook_url
 
-      #
-      # Slack channel to post
-      #
       # @return [String]
-      #
-      def slack_channel
-        Mihari.config.slack_channel || "#general"
-      end
+      attr_reader :channel
 
-      #
-      # Slack webhook URL
-      #
       # @return [String]
-      #
-      def slack_webhook_url
-        Mihari.config.slack_webhook_url
+      attr_reader :username
+
+      def initialize(*args, **kwargs)
+        super(*args, **kwargs)
+
+        @webhook_url = kwargs[:webhook_url] || Mihari.config.slack_webhook_url
+        @channel = kwargs[:channel] || Mihari.config.slack_channel || DEFAULT_CHANNEL
+        @username = DEFAULT_USERNAME
       end
 
       #
-      # Check Slack webhook URL is set
+      # Check webhook URL is set
       #
       # @return [Boolean]
       #
-      def slack_webhook_url?
-        !Mihari.config.slack_webhook_url.nil?
+      def webhook_url?
+        !webhook_url.nil?
       end
 
       #
-      # Check Slack webhook URL is set. Alias of #slack_webhook_url?.
+      # Check webhook URL is set. Alias of #webhook_url?
       #
       # @return [Boolean]
       #
       def valid?
-        slack_webhook_url?
+        webhook_url?
       end
 
       def notifier
-        @notifier ||= ::Slack::Notifier.new(slack_webhook_url, channel: slack_channel, username: DEFAULT_USERNAME)
+        @notifier ||= ::Slack::Notifier.new(webhook_url, channel: channel, username: username)
       end
 
       #
@@ -193,12 +191,6 @@ module Mihari
 
         notifier.post(text: text, attachments: attachments, mrkdwn: true)
       end
-
-      private
-
-      def configuration_keys
-        %w[slack_webhook_url]
-      end
     end
   end
 end

data/lib/mihari/emitters/the_hive.rb

@@ -5,6 +5,19 @@ require "hachi"
 module Mihari
   module Emitters
     class TheHive < Base
+      # @return [String, nil]
+      attr_reader :api_endpoint
+
+      # @return [String, nil]
+      attr_reader :api_key
+
+      def initialize(*args, **kwargs)
+        super(*args, **kwargs)
+
+        @api_endpoint = kwargs[:api_endpoint] || Mihari.config.thehive_api_endpoint
+        @api_key = kwargs[:api_key] || Mihari.config.thehive_api_key
+      end
+
       # @return [Boolean]
       def valid?
         api_endpont? && api_key? && ping?
@@ -30,7 +43,7 @@ module Mihari
       end
 
       def api
-        @api ||= Hachi::API.new(api_endpoint: Mihari.config.thehive_api_endpoint, api_key: Mihari.config.thehive_api_key)
+        @api ||= Hachi::API.new(api_endpoint: api_endpoint, api_key: api_key)
       end
 
       #
@@ -39,16 +52,16 @@ module Mihari
       # @return [Boolean]
       #
       def api_endpont?
-        !Mihari.config.thehive_api_endpoint.nil?
+        !api_endpoint.nil?
       end
 
       #
       # Check whether an API key is set or not
       #
       # @return [Boolean]
-      # ]
+      #
       def api_key?
-        !Mihari.config.thehive_api_key.nil?
+        !api_key.nil?
       end
 
       #
@@ -57,8 +70,7 @@ module Mihari
       # @return [Boolean]
       #
       def ping?
-        base_url = Mihari.config.thehive_api_endpoint
-        base_url = base_url.end_with?("/") ? base_url[0..-2] : base_url
+        base_url = api_endpoint.end_with?("/") ? api_endpoint[0..-2] : api_endpoint
         url = "#{base_url}/index.html"
 
         http = Net::Ping::HTTP.new(url)

data/lib/mihari/entities/rule.rb

@@ -14,20 +14,9 @@ module Mihari
 
     class Rule < Grape::Entity
       expose :id, documentation: { type: String, required: true }
-
       expose :yaml, documentation: { type: String, required: true }
-
-      expose :title, documentation: { type: String, required: true }
-      expose :description, documentation: { type: String, required: true }
-      expose :queries, using: Entities::Query, documentation: { type: Entities::Query, is_array: true, required: true }
-      expose :emitters, using: Entities::Emitter, documentation: { type: Entities::Emitter, is_array: true, required: false }
-      expose :tags, documentation: { type: String, is_array: true }
-      expose :allowed_data_types, documentation: { type: String, is_array: true }, as: :allowedDtaTypes
-      expose :disallowed_data_values, documentation: { type: String, is_array: true }, as: :disallowedDataValues
-      expose :ignore_old_artifacts, documentation: { type: "boolean", required: true }, as: :ignoreOldArtifacts
-      expose :ignore_threshold, documentation: { type: Integer, required: true }, as: :ignoreThreshold
-
       expose :created_at, documentation: { type: DateTime, required: true }, as: :createdAt
+      expose :updated_at, documentation: { type: DateTime, required: true }, as: :updatedAt
     end
 
     class RulesWithPagination < Grape::Entity

data/lib/mihari/errors.rb

@@ -5,6 +5,8 @@ module Mihari
 
   class InvalidInputError < Error; end
 
+  class InvalidArtifactFormatError < Error; end
+
   class RetryableError < Error; end
 
   class FileNotFoundError < Error; end

data/lib/mihari/mixins/configurable.rb

@@ -9,7 +9,9 @@ module Mihari
       # @return [Boolean]
       #
       def configured?
-        configuration_keys.all? { |key| Mihari.config.send(key) }
+        return true if configuration_keys.empty?
+
+        configuration_keys.all? { |key| Mihari.config.send(key) } || api_key?
       end
 
       #
@@ -33,6 +35,15 @@ module Mihari
       def configuration_keys
         []
       end
+
+      private
+
+      def api_key?
+        value = method(:api_key).call
+        !value.nil?
+      rescue NameError
+        true
+      end
     end
   end
 end

data/lib/mihari/mixins/rule.rb

@@ -9,6 +9,10 @@ module Mihari
     module Rule
       include Mixins::Database
 
+      def load_erb_yaml(yaml)
+        YAML.safe_load(ERB.new(yaml).result, permitted_classes: [Date], symbolize_names: true)
+      end
+
       #
       # Load rule into hash
       #
@@ -17,24 +21,24 @@ module Mihari
       # @return [Mihari::Structs::Rule::Rule]
       #
       def load_rule(path_or_id)
-        data = nil
+        yaml = nil
 
-        data = load_data_from_file(path_or_id) if File.exist?(path_or_id)
-        data = load_data_from_db(path_or_id) if data.nil?
+        yaml = load_yaml_from_file(path_or_id) if File.exist?(path_or_id)
+        yaml = load_yaml_from_db(path_or_id) if yaml.nil?
 
-        Structs::Rule::Rule.new(data)
+        Structs::Rule::Rule.from_yaml yaml
       end
 
-      def load_data_from_file(path)
-        return YAML.safe_load(File.read(path), permitted_classes: [Date], symbolize_names: true) if Pathname(path).exist?
+      def load_yaml_from_file(path)
+        return nil unless Pathname(path).exist?
 
-        YAML.safe_load(path, symbolize_names: true)
+        File.read path
       end
 
-      def load_data_from_db(id)
+      def load_yaml_from_db(id)
         with_db_connection do
           rule = Mihari::Rule.find(id)
-          rule.data
+          rule.yaml || rule.symbolized_data.to_yaml
         rescue ActiveRecord::RecordNotFound
           raise ArgumentError, "ID:#{id} is not found in the database"
         end
@@ -58,16 +62,9 @@ module Mihari
       # @return [String] A template for rule
       #
       def rule_template
-        # Use ERB to fill created_on and updated_on with Date.today
-        data = File.read(File.expand_path("../templates/rule.yml.erb", __dir__))
-        template = ERB.new(data)
-        data = template.result
-
-        # validate the template of rule for just in case
-        hashed_data = YAML.safe_load(data, permitted_classes: [Date], symbolize_names: true)
-        Structs::Rule::Rule.new(hashed_data)
-
-        data
+        yaml = File.read(File.expand_path("../templates/rule.yml.erb", __dir__))
+        Structs::Rule::Rule.from_yaml yaml
+        yaml
       end
 
       #

data/lib/mihari/models/artifact.rb

@@ -25,8 +25,13 @@ module Mihari
 
     attr_accessor :tags
 
-    def initialize(attributes)
-      super
+    def initialize(*args, **kwargs)
+      attrs = args.first || kwargs
+      data_ = attrs[:data]
+
+      raise InvalidArtifactFormatError if data_.is_a?(Array) || data_.is_a?(Hash)
+
+      super(*args, **kwargs)
 
       self.data_type = TypeChecker.type(data)
       self.tags = []

data/lib/mihari/models/rule.rb

@@ -1,18 +1,27 @@
 # frozen_string_literal: true
 
+require "yaml"
+
 module Mihari
   class Rule < ActiveRecord::Base
     has_many :alerts, foreign_key: :source
 
+    def symbolized_data
+      @symbolized_data ||= data.deep_symbolize_keys
+    end
+
     #
     # Returns a hash representative
     #
     # @return [Hash]
     #
     def to_h
-      symbolized_data = data.deep_symbolize_keys
-      h = { id: id, created_at: created_at, yaml: data.to_yaml }
-      h.merge symbolized_data
+      {
+        id: id,
+        yaml: yaml || data.to_yaml,
+        created_at: created_at,
+        updated_at: updated_at
+      }
     end
 
     class << self

data/lib/mihari/schemas/analyzer.rb

@@ -2,19 +2,98 @@
 
 module Mihari
   module Schemas
-    AnalyzerRun = Dry::Schema.Params do
-      required(:title).value(:string)
-      required(:description).value(:string)
-      required(:source).value(:string)
-      required(:artifacts).value(array[:string])
+    AnalyzerOptions = Dry::Schema.Params do
+      optional(:interval).value(:integer)
+    end
+
+    AnalyzerWithoutAPIKey = Dry::Schema.Params do
+      required(:analyzer).value(Types::String.enum("crtsh", "dnpedia", "dnstwister"))
+      required(:query).value(:string)
+      optional(:options).hash(AnalyzerOptions)
+    end
+
+    AnalyzerWithAPIKey = Dry::Schema.Params do
+      required(:analyzer).value(
+        Types::String.enum(
+          "binaryedge",
+          "greynoise",
+          "onyphe",
+          "otx",
+          "pulsedive",
+          "securitytrails",
+          "shodan",
+          "st",
+          "virustotal_intelligence",
+          "virustotal",
+          "vt_intel",
+          "vt"
+        )
+      )
+      required(:query).value(:string)
+      optional(:api_key).value(:string)
+      optional(:options).hash(AnalyzerOptions)
+    end
+
+    Censys = Dry::Schema.Params do
+      required(:analyzer).value(Types::String.enum("censys"))
+      required(:query).value(:string)
+      optional(:id).value(:string)
+      optional(:secret).value(:string)
+      optional(:options).hash(AnalyzerOptions)
+    end
+
+    CIRCL = Dry::Schema.Params do
+      required(:analyzer).value(Types::String.enum("circl"))
+      required(:query).value(:string)
+      optional(:username).value(:string)
+      optional(:password).value(:string)
+      optional(:options).hash(AnalyzerOptions)
+    end
+
+    PassiveTotal = Dry::Schema.Params do
+      required(:analyzer).value(Types::String.enum("passivetotal", "pt"))
+      required(:query).value(:string)
+      optional(:username).value(:string)
+      optional(:api_key).value(:string)
+      optional(:options).hash(AnalyzerOptions)
+    end
+
+    Spyse = Dry::Schema.Params do
+      required(:analyzer).value(Types::String.enum("spyse"))
+      required(:query).value(:string)
+      required(:type).value(Types::String.enum("ip", "domain"))
+      optional(:options).hash(AnalyzerOptions)
+    end
+
+    ZoomEye = Dry::Schema.Params do
+      required(:analyzer).value(Types::String.enum("zoomeye"))
+      required(:query).value(:string)
+      required(:type).value(Types::String.enum("host", "web"))
+      optional(:options).hash(AnalyzerOptions)
+    end
+
+    Crtsh = Dry::Schema.Params do
+      required(:analyzer).value(Types::String.enum("crtsh"))
+      required(:query).value(:string)
+      optional(:exclude_expired).value(:bool).default(true)
+      optional(:options).hash(AnalyzerOptions)
+    end
 
-      optional(:tags).value(array[:string]).default([])
-      optional(:ignoreOldArtifacts).value(:bool).default(false)
-      optional(:ignoreThreshold).value(:integer).default(0)
+    Urlscan = Dry::Schema.Params do
+      required(:analyzer).value(Types::String.enum("urlscan"))
+      required(:query).value(:string)
+      optional(:options).hash(AnalyzerOptions)
     end
 
-    class AnalyzerRunContract < Dry::Validation::Contract
-      params(AnalyzerRun)
+    Feed = Dry::Schema.Params do
+      required(:analyzer).value(Types::String.enum("feed"))
+      required(:query).value(:string)
+      required(:selector).value(:string)
+      optional(:http_request_method).value(Types::HttpRequestMethods).default("GET")
+      optional(:http_request_headers).value(:hash).default({})
+      optional(:http_request_payload).value(:hash).default({})
+      optional(:http_request_payload_type).value(Types::HttpRequestPayloadTypes)
+      optional(:options).hash(AnalyzerOptions)
     end
   end
 end

data/lib/mihari/schemas/emitter.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Schemas
+    Emitter = Dry::Schema.Params do
+      required(:emitter).value(Types::EmitterTypes)
+    end
+
+    MISP = Dry::Schema.Params do
+      required(:emitter).value(Types::String.enum("misp"))
+      optional(:api_endpoint).value(:string)
+      optional(:api_key).value(:string)
+    end
+
+    TheHive = Dry::Schema.Params do
+      required(:emitter).value(Types::String.enum("the_hive"))
+      optional(:api_endpoint).value(:string)
+      optional(:api_key).value(:string)
+    end
+
+    Slack = Dry::Schema.Params do
+      required(:emitter).value(Types::String.enum("slack"))
+      optional(:webhook_url).value(:string)
+      optional(:channel).value(:string)
+    end
+
+    HTTP = Dry::Schema.Params do
+      required(:emitter).value(Types::String.enum("http"))
+      required(:url).value(:string)
+      optional(:http_request_method).value(Types::HttpRequestMethods).default("POST")
+      optional(:http_request_headers).value(:hash).default({})
+      optional(:template).value(:string)
+    end
+  end
+end

data/lib/mihari/schemas/rule.rb

@@ -1,67 +1,10 @@
 # frozen_string_literal: true
 
+require "mihari/schemas/analyzer"
+require "mihari/schemas/emitter"
+
 module Mihari
   module Schemas
-    AnalyzerOptions = Dry::Schema.Params do
-      optional(:interval).value(:integer)
-    end
-
-    Analyzer = Dry::Schema.Params do
-      required(:analyzer).value(Types::AnalyzerTypes)
-      required(:query).value(:string)
-      optional(:options).hash(AnalyzerOptions)
-    end
-
-    Spyse = Dry::Schema.Params do
-      required(:analyzer).value(Types::String.enum("spyse"))
-      required(:query).value(:string)
-      required(:type).value(Types::String.enum("ip", "domain"))
-      optional(:options).hash(AnalyzerOptions)
-    end
-
-    ZoomEye = Dry::Schema.Params do
-      required(:analyzer).value(Types::String.enum("zoomeye"))
-      required(:query).value(:string)
-      required(:type).value(Types::String.enum("host", "web"))
-      optional(:options).hash(AnalyzerOptions)
-    end
-
-    Crtsh = Dry::Schema.Params do
-      required(:analyzer).value(Types::String.enum("crtsh"))
-      required(:query).value(:string)
-      optional(:exclude_expired).value(:bool).default(true)
-      optional(:options).hash(AnalyzerOptions)
-    end
-
-    Urlscan = Dry::Schema.Params do
-      required(:analyzer).value(Types::String.enum("urlscan"))
-      required(:query).value(:string)
-      optional(:options).hash(AnalyzerOptions)
-    end
-
-    Feed = Dry::Schema.Params do
-      required(:analyzer).value(Types::String.enum("feed"))
-      required(:query).value(:string)
-      required(:selector).value(:string)
-      optional(:http_request_method).value(Types::HttpRequestMethods).default("GET")
-      optional(:http_request_headers).value(:hash).default({})
-      optional(:http_request_payload).value(:hash).default({})
-      optional(:http_request_payload_type).value(Types::HttpRequestPayloadTypes)
-      optional(:options).hash(AnalyzerOptions)
-    end
-
-    Emitter = Dry::Schema.Params do
-      required(:emitter).value(Types::EmitterTypes)
-    end
-
-    HTTP = Dry::Schema.Params do
-      required(:emitter).value(Types::String.enum("http"))
-      required(:url).value(:string)
-      optional(:http_request_method).value(Types::HttpRequestMethods).default("POST")
-      optional(:http_request_headers).value(:hash).default({})
-      optional(:template).value(:string)
-    end
-
     Rule = Dry::Schema.Params do
       required(:title).value(:string)
       required(:description).value(:string)
@@ -73,9 +16,9 @@ module Mihari
       optional(:created_on).value(:date)
       optional(:updated_on).value(:date)
 
-      required(:queries).value(:array).each { Analyzer | Spyse | ZoomEye | Urlscan | Crtsh | Feed }
+      required(:queries).value(:array).each { AnalyzerWithoutAPIKey | AnalyzerWithAPIKey | Censys | CIRCL | PassiveTotal | Spyse | ZoomEye | Urlscan | Crtsh | Feed }
 
-      optional(:emitters).value(:array).each { Emitter | HTTP }
+      optional(:emitters).value(:array).each { Emitter | MISP | TheHive | Slack | HTTP }
 
       optional(:allowed_data_types).value(array[Types::DataTypes]).default(ALLOWED_DATA_TYPES)
       optional(:disallowed_data_values).value(array[:string]).default([])

data/lib/mihari/structs/rule.rb

@@ -27,10 +27,19 @@ module Mihari
       end
 
       class Rule
-        attr_reader :data, :errors
+        # @return [Hash]
+        attr_reader :data
 
-        def initialize(data)
+        # @return [String]
+        attr_reader :yaml
+
+        # @return [Array, nil]
+        attr_reader :errors
+
+        def initialize(data, yaml)
           @data = data.deep_symbolize_keys
+          @yaml = yaml
+
           @errors = nil
           @no_method_error = nil
 
@@ -76,7 +85,7 @@ module Mihari
 
           raise RuleValidationError, error_messages.join("\n") if errors?
 
-          raise RuleValidationError, "Something wrong with queries" unless @no_method_error.nil?
+          raise RuleValidationError, "Something wrong with queries or emitters." unless @no_method_error.nil?
         end
 
         def [](key)
@@ -109,16 +118,20 @@ module Mihari
         #
         def to_model
           rule = Mihari::Rule.find(id)
+
           rule.title = title
           rule.description = description
           rule.data = data
+          rule.yaml = yaml
+
           rule
         rescue ActiveRecord::RecordNotFound
           Mihari::Rule.new(
             id: id,
             title: title,
             description: description,
-            data: data
+            data: data,
+            yaml: yaml
           )
         end
 
@@ -143,13 +156,28 @@ module Mihari
         end
 
         class << self
+          include Mixins::Rule
+
           #
           # @param [Mihari::Rule] model
           #
           # @return [Mihari::Structs::Rule::Rule]
           #
           def from_model(model)
-            Structs::Rule::Rule.new(model.data)
+            data = model.data.deep_symbolize_keys
+            data[:id] = model.id unless data.key?(:id)
+
+            Structs::Rule::Rule.new(data, model.yaml)
+          end
+
+          #
+          # @param [String] yaml
+          #
+          # @return [Mihari::Structs::Rule::Rule]
+          #
+          def from_yaml(yaml)
+            data = load_erb_yaml(yaml)
+            Structs::Rule::Rule.new(data, yaml)
           end
         end
       end

data/lib/mihari/types.rb

@@ -16,36 +16,11 @@ module Mihari
 
     UrlscanDataTypes = Types::String.enum("ip", "domain", "url")
 
-    AnalyzerTypes = Types::String.enum(
-      "binaryedge",
-      "censys",
-      "circl",
-      "dnpedia",
-      "dnstwister",
-      "greynoise",
-      "onyphe",
-      "otx",
-      "passivetotal",
-      "pt",
-      "pulsedive",
-      "securitytrails",
-      "shodan",
-      "st",
-      "virustotal_intelligence",
-      "virustotal",
-      "vt_intel",
-      "vt"
-    )
-
     HttpRequestMethods = Types::String.enum("GET", "POST")
     HttpRequestPayloadTypes = Types::String.enum("application/json", "application/x-www-form-urlencoded")
 
     EmitterTypes = Types::String.enum(
       "database",
-      "http",
-      "misp",
-      "slack",
-      "the_hive",
       "webhook"
     )
   end

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "4.3.0"
+  VERSION = "4.4.0"
 end