mihari 4.11.0 → 5.0.0

data/lib/mihari/entities/config.rb

@@ -2,15 +2,11 @@
 
 module Mihari
   module Entities
-    class ConfigStatus < Grape::Entity
+    class Config < Grape::Entity
+      expose :name, documentation: { type: String, required: true }
       expose :type, documentation: { type: String, required: true }
       expose :values, documentation: { type: String, is_array: true, required: true }
       expose :is_configured, documentation: { type: Grape::API::Boolean, required: true }, as: :isConfigured
     end
-
-    class Config < Grape::Entity
-      expose :name, documentation: { type: String, required: true }
-      expose :status, using: Entities::ConfigStatus, documentation: { type: ConfigStatus, required: true }
-    end
   end
 end

data/lib/mihari/entities/rule.rb

@@ -14,9 +14,13 @@ module Mihari
 
     class Rule < Grape::Entity
       expose :id, documentation: { type: String, required: true }
+      expose :title, documentation: { type: String, required: true }
+      expose :description, documentation: { type: String, required: true }
       expose :yaml, documentation: { type: String, required: true }
       expose :created_at, documentation: { type: DateTime, required: true }, as: :createdAt
       expose :updated_at, documentation: { type: DateTime, required: true }, as: :updatedAt
+
+      expose :tags, using: Entities::Tag, documentation: { type: Entities::Tag, is_array: true, required: true }
     end
 
     class RulesWithPagination < Grape::Entity
@@ -25,5 +29,9 @@ module Mihari
       expose :current_page, documentation: { type: Integer, required: true }, as: :currentPage
       expose :page_size, documentation: { type: Integer, required: true }, as: :pageSize
     end
+
+    class RuleIDs < Grape::Entity
+      expose :rule_ids, documentation: { type: Array[String], required: true }, as: :ruleIds
+    end
   end
 end

data/lib/mihari/http.rb

@@ -4,10 +4,10 @@ require "insensitive_hash"
 
 module Mihari
   class HTTP
-    attr_reader :uri, :headers, :payload
+    attr_reader :url, :headers, :payload
 
-    def initialize(uri, headers: {}, payload: {})
-      @uri = uri.is_a?(URI) ? uri : URI(uri.to_s)
+    def initialize(url, headers: {}, payload: {})
+      @url = url.is_a?(URI) ? url : URI(url.to_s)
       @headers = headers.insensitive
       @payload = payload
     end
@@ -18,10 +18,10 @@ module Mihari
     # @return [Net::HTTPResponse]
     #
     def get
-      new_uri = uri.deep_dup
-      new_uri.query = Addressable::URI.form_encode(payload) unless payload.empty?
+      new_url = url.deep_dup
+      new_url.query = Addressable::URI.form_encode(payload) unless payload.empty?
 
-      get = Net::HTTP::Get.new(new_uri)
+      get = Net::HTTP::Get.new(new_url)
       request get
     end
 
@@ -31,7 +31,7 @@ module Mihari
     # @return [Net::HTTPResponse]
     #
     def post
-      post = Net::HTTP::Post.new(uri)
+      post = Net::HTTP::Post.new(url)
 
       case content_type
       when "application/json"
@@ -44,13 +44,13 @@ module Mihari
     end
 
     class << self
-      def get(uri, headers: {}, params: {})
-        client = new(uri, headers: headers, payload: params)
+      def get(url, headers: {}, params: {})
+        client = new(url, headers: headers, payload: params)
         client.get
       end
 
-      def post(uri, headers: {}, payload: {})
-        client = new(uri, headers: headers, payload: payload)
+      def post(url, headers: {}, payload: {})
+        client = new(url, headers: headers, payload: payload)
         client.post
       end
     end
@@ -67,7 +67,7 @@ module Mihari
     # @return [Hahs]
     #
     def https_options
-      return { use_ssl: true } if uri.scheme == "https"
+      return { use_ssl: true } if url.scheme == "https"
 
       {}
     end
@@ -80,7 +80,7 @@ module Mihari
     # @return [Net::HTTPResponse]
     #
     def request(req)
-      Net::HTTP.start(uri.host, uri.port, https_options) do |http|
+      Net::HTTP.start(url.host, url.port, https_options) do |http|
         # set headers
         headers.each do |k, v|
           req[k] = v

data/lib/mihari/mixins/{disallowed_data_value.rb → falsepositive.rb}

@@ -2,24 +2,24 @@
 
 module Mihari
   module Mixins
-    module DisallowedDataValue
+    module FalsePositive
       include Memist::Memoizable
 
       #
-      # Normalize a value as a disallowed data value
+      # Normalize a falsepositive value
       #
-      # @param [String] value Data value
+      # @param [String] value
       #
-      # @return [String, Regexp] Normalized value
+      # @return [String, Regexp]
       #
-      def normalize_disallowed_data_value(value)
+      def normalize_falsepositive(value)
         return value if !value.start_with?("/") || !value.end_with?("/")
 
         # if a value is surrounded by slashes, take it as a regexp
         value_without_slashes = value[1..-2]
         Regexp.compile value_without_slashes.to_s
       end
-      memoize :normalize_disallowed_data_value
+      memoize :normalize_falsepositive
 
       #
       # Check whetehr a value is valid format as a disallowed data value
@@ -28,9 +28,9 @@ module Mihari
       #
       # @return [Boolean] true if it is valid, otherwise false
       #
-      def valid_disallowed_data_value?(value)
+      def valid_falsepositive?(value)
         begin
-          normalize_disallowed_data_value value
+          normalize_falsepositive value
         rescue RegexpError
           return false
         end

data/lib/mihari/models/alert.rb

@@ -28,20 +28,7 @@ module Mihari
         relation = build_relation(filter.without_pagination)
 
         alert_ids = relation.limit(limit).offset(offset).order(id: :desc).pluck(:id).uniq
-        eager_load(
-          {
-            artifacts: [
-              :autonomous_system,
-              :geolocation,
-              :whois_record,
-              :dns_records,
-              :reverse_dns_names,
-              :cpes,
-              :ports
-            ]
-          },
-          :tags
-        ).where(id: [alert_ids]).order(id: :desc)
+        eager_load(:artifacts, :tags).where(id: [alert_ids]).order(id: :desc)
       end
 
       #
@@ -83,7 +70,7 @@ module Mihari
         relation = relation.where(artifacts: { id: artifact_ids }) unless artifact_ids.empty?
         relation = relation.where(tags: { name: filter.tag_name }) if filter.tag_name
 
-        relation = relation.where(source: filter.source) if filter.source
+        relation = relation.where(rule_id: filter.rule_id) if filter.rule_id
         relation = relation.where(title: filter.title) if filter.title
 
         relation = relation.where("description LIKE ?", "%#{filter.description}%") if filter.description

data/lib/mihari/models/artifact.rb

@@ -25,8 +25,12 @@ module Mihari
 
     validates_with ArtifactValidator
 
+    # @return [Array<Mihari::Tag>] Tags
     attr_accessor :tags
 
+    # @return [String, nil] Rule ID
+    attr_accessor :rule_id
+
     def initialize(*args, **kwargs)
       attrs = args.first || kwargs
       data_ = attrs[:data]
@@ -36,27 +40,34 @@ module Mihari
       super(*args, **kwargs)
 
       self.data_type = TypeChecker.type(data)
-      self.tags = []
+
+      @tags = []
+      @rule_id = ""
     end
 
     #
     # Check uniqueness of artifact
     #
-    # @param [Boolean] ignore_old_artifacts
-    # @param [Integer] ignore_threshold
+    # @param [Time, nil] base_time Base time to check decaying
+    # @param [Integer, nil] artifact_lifetime Artifact lifetime (TTL) in seconds
     #
     # @return [Boolean] true if it is unique. Otherwise false.
     #
-    def unique?(ignore_old_artifacts: false, ignore_threshold: 0)
-      artifact = self.class.where(data: data).order(created_at: :desc).first
+    def unique?(base_time: nil, artifact_lifetime: nil)
+      artifact = self.class.joins(:alert).where(
+        data: data,
+        alert: { rule_id: rule_id }
+      ).order(created_at: :desc).first
       return true if artifact.nil?
 
-      return false unless ignore_old_artifacts
+      # check whetehr the artifact is decayed or not
+      return false if artifact_lifetime.nil?
+
+      # use the current UTC time if base_time is not given (for testing)
+      base_time ||= Time.now.utc
 
-      days_before = (-ignore_threshold).days.from_now.utc
-      # if an artifact is created before {ignore_threshold} days, ignore it
-      #                           within {ignore_threshold} days, do not ignore it
-      artifact.created_at < days_before
+      decayed_at = base_time - (artifact_lifetime || -1).seconds
+      artifact.created_at < decayed_at
     end
 
     #
@@ -139,14 +150,14 @@ module Mihari
       whois: [
         :enrich_whois
       ],
-      ipinfo: [
-        :enrich_autonomous_system,
-        :enrich_geolocation
+      ipinfo: %i[
+        enrich_autonomous_system
+        enrich_geolocation
       ],
-      shodan: [
-        :enrich_ports,
-        :enrich_cpes,
-        :enrich_reverse_dns
+      shodan: %i[
+        enrich_ports
+        enrich_cpes
+        enrich_reverse_dns
       ],
       google_public_dns: [
         :enrich_dns

data/lib/mihari/models/rule.rb

@@ -4,24 +4,18 @@ require "yaml"
 
 module Mihari
   class Rule < ActiveRecord::Base
-    has_many :alerts, foreign_key: :source
+    has_many :alerts, dependent: :destroy
 
     def symbolized_data
       @symbolized_data ||= data.deep_symbolize_keys
     end
 
-    #
-    # Returns a hash representative
-    #
-    # @return [Hash]
-    #
-    def to_h
-      {
-        id: id,
-        yaml: yaml || data.to_yaml,
-        created_at: created_at,
-        updated_at: updated_at
-      }
+    def yaml
+      data.to_yaml
+    end
+
+    def tags
+      (data["tags"] || []).map { |tag| { name: tag } }
     end
 
     class << self

data/lib/mihari/schemas/emitter.rb

@@ -2,20 +2,18 @@
 
 module Mihari
   module Schemas
-    Emitter = Dry::Schema.Params do
-      required(:emitter).value(Types::EmitterTypes)
+    Database = Dry::Schema.Params do
+      required(:emitter).value(Types::String.enum("database"))
     end
 
     MISP = Dry::Schema.Params do
       required(:emitter).value(Types::String.enum("misp"))
-      optional(:api_endpoint).value(:string)
       optional(:url).value(:string)
       optional(:api_key).value(:string)
     end
 
     TheHive = Dry::Schema.Params do
       required(:emitter).value(Types::String.enum("the_hive"))
-      optional(:api_endpoint).value(:string)
       optional(:url).value(:string)
       optional(:api_key).value(:string)
       optional(:api_version).value(Types::String.enum("v4", "v5")).default("v4")
@@ -27,11 +25,11 @@ module Mihari
       optional(:channel).value(:string)
     end
 
-    HTTP = Dry::Schema.Params do
-      required(:emitter).value(Types::String.enum("http"))
+    Webhook = Dry::Schema.Params do
+      required(:emitter).value(Types::String.enum("webhook"))
       required(:url).value(:string)
-      optional(:http_request_method).value(Types::HTTPRequestMethods).default("POST")
-      optional(:http_request_headers).value(:hash).default({})
+      optional(:method).value(Types::HTTPRequestMethods).default("POST")
+      optional(:headers).value(:hash).default({})
       optional(:template).value(:string)
     end
   end

data/lib/mihari/schemas/rule.rb

@@ -7,18 +7,17 @@ require "mihari/schemas/enricher"
 module Mihari
   module Schemas
     Rule = Dry::Schema.Params do
+      required(:id).value(:string)
       required(:title).value(:string)
       required(:description).value(:string)
 
       optional(:tags).value(array[:string]).default([])
-      optional(:id).value(:string)
 
-      optional(:status).value(:string)
-
-      optional(:related).value(array[:string])
+      optional(:author).value(:string)
       optional(:references).value(array[:string])
+      optional(:related).value(array[:string])
+      optional(:status).value(:string)
 
-      optional(:author).value(:string)
       optional(:created_on).value(:date)
       optional(:updated_on).value(:date)
 
@@ -26,15 +25,14 @@ module Mihari
         AnalyzerWithoutAPIKey | AnalyzerWithAPIKey | Censys | CIRCL | PassiveTotal | ZoomEye | Urlscan | Crtsh | Feed
       end
 
-      optional(:emitters).value(:array).each { Emitter | MISP | TheHive | Slack | HTTP }
+      optional(:emitters).value(:array).each { Database | MISP | TheHive | Slack | Webhook }
 
       optional(:enrichers).value(:array).each(Enricher)
 
-      optional(:allowed_data_types).value(array[Types::DataTypes]).default(ALLOWED_DATA_TYPES)
-      optional(:disallowed_data_values).value(array[:string]).default([])
+      optional(:data_types).value(array[Types::DataTypes]).default(DEFAULT_DATA_TYPES)
+      optional(:falsepositives).value(array[:string]).default([])
 
-      optional(:ignore_old_artifacts).value(:bool).default(false)
-      optional(:ignore_threshold).value(:integer).default(0)
+      optional(:artifact_lifetime).value(:integer)
 
       before(:key_coercer) do |result|
         # it looks like that dry-schema v1.9.1 has an issue with setting an array of schemas as a default value
@@ -53,13 +51,13 @@ module Mihari
     end
 
     class RuleContract < Dry::Validation::Contract
-      include Mihari::Mixins::DisallowedDataValue
+      include Mihari::Mixins::FalsePositive
 
       params(Rule)
 
-      rule(:disallowed_data_values) do
+      rule(:falsepositives) do
         value.each do |v|
-          key.failure("#{v} is not a valid format.") unless valid_disallowed_data_value?(v)
+          key.failure("#{v} is not a valid format.") unless valid_falsepositive?(v)
         end
       end
     end

data/lib/mihari/structs/config.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Structs
+    class Config < Dry::Struct
+      attribute :name, Types::String
+      attribute :type, Types::String
+      attribute :is_configured, Types::Bool
+      attribute :values, Types.Array(Types::Hash).optional
+
+      #
+      # @param [Class<Mihari::Analyzers::Base>, Class<Mihari::Emitters::Base>] klass
+      #
+      # @return [Mihari::Structs::Config, nil] config
+      #
+      def self.from_class(klass)
+        return nil if klass == Mihari::Analyzers::Rule
+
+        name = klass.to_s.split("::").last.to_s
+
+        is_analyzer = klass.ancestors.include?(Mihari::Analyzers::Base)
+        is_emitter = klass.ancestors.include?(Mihari::Emitters::Base)
+        is_enricher = klass.ancestors.include?(Mihari::Enrichers::Base)
+
+        type = "Analyzer"
+        type = "Emitter" if is_emitter
+        type = "Enricher" if is_enricher
+
+        begin
+          instance = is_analyzer ? klass.new("dummy") : klass.new
+          is_configured = instance.configured?
+          values = instance.configuration_values
+
+          new(name: name, values: values, is_configured: is_configured, type: type)
+        rescue ArgumentError => _e
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/mihari/structs/filters.rb

@@ -7,7 +7,7 @@ module Mihari
         class SearchFilter < Dry::Struct
           attribute? :artifact_data, Types::String.optional
           attribute? :description, Types::String.optional
-          attribute? :source, Types::String.optional
+          attribute? :rule_id, Types::String.optional
           attribute? :tag_name, Types::String.optional
           attribute? :title, Types::String.optional
           attribute? :from_at, Types::DateTime.optional
@@ -30,7 +30,7 @@ module Mihari
               artifact_data: artifact_data,
               description: description,
               from_at: from_at,
-              source: source,
+              rule_id: rule_id,
               tag_name: tag_name,
               title: title,
               to_at: to_at,