mihari 4.12.0 → 5.0.0

data/lib/mihari/models/artifact.rb

@@ -25,8 +25,12 @@ module Mihari
 
     validates_with ArtifactValidator
 
+    # @return [Array<Mihari::Tag>] Tags
     attr_accessor :tags
 
+    # @return [String, nil] Rule ID
+    attr_accessor :rule_id
+
     def initialize(*args, **kwargs)
       attrs = args.first || kwargs
       data_ = attrs[:data]
@@ -36,27 +40,34 @@ module Mihari
       super(*args, **kwargs)
 
       self.data_type = TypeChecker.type(data)
-      self.tags = []
+
+      @tags = []
+      @rule_id = ""
     end
 
     #
     # Check uniqueness of artifact
     #
-    # @param [Boolean] ignore_old_artifacts
-    # @param [Integer] ignore_threshold
+    # @param [Time, nil] base_time Base time to check decaying
+    # @param [Integer, nil] artifact_lifetime Artifact lifetime (TTL) in seconds
     #
     # @return [Boolean] true if it is unique. Otherwise false.
     #
-    def unique?(ignore_old_artifacts: false, ignore_threshold: 0)
-      artifact = self.class.where(data: data).order(created_at: :desc).first
+    def unique?(base_time: nil, artifact_lifetime: nil)
+      artifact = self.class.joins(:alert).where(
+        data: data,
+        alert: { rule_id: rule_id }
+      ).order(created_at: :desc).first
       return true if artifact.nil?
 
-      return false unless ignore_old_artifacts
+      # check whetehr the artifact is decayed or not
+      return false if artifact_lifetime.nil?
+
+      # use the current UTC time if base_time is not given (for testing)
+      base_time ||= Time.now.utc
 
-      days_before = (-ignore_threshold).days.from_now.utc
-      # if an artifact is created before {ignore_threshold} days, ignore it
-      #                           within {ignore_threshold} days, do not ignore it
-      artifact.created_at < days_before
+      decayed_at = base_time - (artifact_lifetime || -1).seconds
+      artifact.created_at < decayed_at
     end
 
     #
@@ -139,14 +150,14 @@ module Mihari
       whois: [
         :enrich_whois
       ],
-      ipinfo: [
-        :enrich_autonomous_system,
-        :enrich_geolocation
+      ipinfo: %i[
+        enrich_autonomous_system
+        enrich_geolocation
       ],
-      shodan: [
-        :enrich_ports,
-        :enrich_cpes,
-        :enrich_reverse_dns
+      shodan: %i[
+        enrich_ports
+        enrich_cpes
+        enrich_reverse_dns
       ],
       google_public_dns: [
         :enrich_dns

data/lib/mihari/models/rule.rb

@@ -4,24 +4,18 @@ require "yaml"
 
 module Mihari
   class Rule < ActiveRecord::Base
-    has_many :alerts, foreign_key: :source
+    has_many :alerts, dependent: :destroy
 
     def symbolized_data
       @symbolized_data ||= data.deep_symbolize_keys
     end
 
-    #
-    # Returns a hash representative
-    #
-    # @return [Hash]
-    #
-    def to_h
-      {
-        id: id,
-        yaml: yaml || data.to_yaml,
-        created_at: created_at,
-        updated_at: updated_at
-      }
+    def yaml
+      data.to_yaml
+    end
+
+    def tags
+      (data["tags"] || []).map { |tag| { name: tag } }
     end
 
     class << self

data/lib/mihari/schemas/emitter.rb

@@ -2,20 +2,18 @@
 
 module Mihari
   module Schemas
-    Emitter = Dry::Schema.Params do
-      required(:emitter).value(Types::EmitterTypes)
+    Database = Dry::Schema.Params do
+      required(:emitter).value(Types::String.enum("database"))
     end
 
     MISP = Dry::Schema.Params do
       required(:emitter).value(Types::String.enum("misp"))
-      optional(:api_endpoint).value(:string)
       optional(:url).value(:string)
       optional(:api_key).value(:string)
     end
 
     TheHive = Dry::Schema.Params do
       required(:emitter).value(Types::String.enum("the_hive"))
-      optional(:api_endpoint).value(:string)
       optional(:url).value(:string)
       optional(:api_key).value(:string)
       optional(:api_version).value(Types::String.enum("v4", "v5")).default("v4")
@@ -27,11 +25,11 @@ module Mihari
       optional(:channel).value(:string)
     end
 
-    HTTP = Dry::Schema.Params do
-      required(:emitter).value(Types::String.enum("http"))
+    Webhook = Dry::Schema.Params do
+      required(:emitter).value(Types::String.enum("webhook"))
       required(:url).value(:string)
-      optional(:http_request_method).value(Types::HTTPRequestMethods).default("POST")
-      optional(:http_request_headers).value(:hash).default({})
+      optional(:method).value(Types::HTTPRequestMethods).default("POST")
+      optional(:headers).value(:hash).default({})
       optional(:template).value(:string)
     end
   end

data/lib/mihari/schemas/rule.rb

@@ -7,18 +7,17 @@ require "mihari/schemas/enricher"
 module Mihari
   module Schemas
     Rule = Dry::Schema.Params do
+      required(:id).value(:string)
       required(:title).value(:string)
       required(:description).value(:string)
 
       optional(:tags).value(array[:string]).default([])
-      optional(:id).value(:string)
 
-      optional(:status).value(:string)
-
-      optional(:related).value(array[:string])
+      optional(:author).value(:string)
       optional(:references).value(array[:string])
+      optional(:related).value(array[:string])
+      optional(:status).value(:string)
 
-      optional(:author).value(:string)
       optional(:created_on).value(:date)
       optional(:updated_on).value(:date)
 
@@ -26,15 +25,14 @@ module Mihari
         AnalyzerWithoutAPIKey | AnalyzerWithAPIKey | Censys | CIRCL | PassiveTotal | ZoomEye | Urlscan | Crtsh | Feed
       end
 
-      optional(:emitters).value(:array).each { Emitter | MISP | TheHive | Slack | HTTP }
+      optional(:emitters).value(:array).each { Database | MISP | TheHive | Slack | Webhook }
 
       optional(:enrichers).value(:array).each(Enricher)
 
-      optional(:allowed_data_types).value(array[Types::DataTypes]).default(ALLOWED_DATA_TYPES)
-      optional(:disallowed_data_values).value(array[:string]).default([])
+      optional(:data_types).value(array[Types::DataTypes]).default(DEFAULT_DATA_TYPES)
+      optional(:falsepositives).value(array[:string]).default([])
 
-      optional(:ignore_old_artifacts).value(:bool).default(false)
-      optional(:ignore_threshold).value(:integer).default(0)
+      optional(:artifact_lifetime).value(:integer)
 
       before(:key_coercer) do |result|
         # it looks like that dry-schema v1.9.1 has an issue with setting an array of schemas as a default value
@@ -53,13 +51,13 @@ module Mihari
     end
 
     class RuleContract < Dry::Validation::Contract
-      include Mihari::Mixins::DisallowedDataValue
+      include Mihari::Mixins::FalsePositive
 
       params(Rule)
 
-      rule(:disallowed_data_values) do
+      rule(:falsepositives) do
         value.each do |v|
-          key.failure("#{v} is not a valid format.") unless valid_disallowed_data_value?(v)
+          key.failure("#{v} is not a valid format.") unless valid_falsepositive?(v)
         end
       end
     end

data/lib/mihari/structs/config.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Structs
+    class Config < Dry::Struct
+      attribute :name, Types::String
+      attribute :type, Types::String
+      attribute :is_configured, Types::Bool
+      attribute :values, Types.Array(Types::Hash).optional
+
+      #
+      # @param [Class<Mihari::Analyzers::Base>, Class<Mihari::Emitters::Base>] klass
+      #
+      # @return [Mihari::Structs::Config, nil] config
+      #
+      def self.from_class(klass)
+        return nil if klass == Mihari::Analyzers::Rule
+
+        name = klass.to_s.split("::").last.to_s
+
+        is_analyzer = klass.ancestors.include?(Mihari::Analyzers::Base)
+        is_emitter = klass.ancestors.include?(Mihari::Emitters::Base)
+        is_enricher = klass.ancestors.include?(Mihari::Enrichers::Base)
+
+        type = "Analyzer"
+        type = "Emitter" if is_emitter
+        type = "Enricher" if is_enricher
+
+        begin
+          instance = is_analyzer ? klass.new("dummy") : klass.new
+          is_configured = instance.configured?
+          values = instance.configuration_values
+
+          new(name: name, values: values, is_configured: is_configured, type: type)
+        rescue ArgumentError => _e
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/mihari/structs/filters.rb

@@ -7,7 +7,7 @@ module Mihari
         class SearchFilter < Dry::Struct
           attribute? :artifact_data, Types::String.optional
           attribute? :description, Types::String.optional
-          attribute? :source, Types::String.optional
+          attribute? :rule_id, Types::String.optional
           attribute? :tag_name, Types::String.optional
           attribute? :title, Types::String.optional
           attribute? :from_at, Types::DateTime.optional
@@ -30,7 +30,7 @@ module Mihari
               artifact_data: artifact_data,
               description: description,
               from_at: from_at,
-              source: source,
+              rule_id: rule_id,
               tag_name: tag_name,
               title: title,
               to_at: to_at,

data/lib/mihari/structs/rule.rb

@@ -4,6 +4,7 @@ require "date"
 require "erb"
 require "json"
 require "pathname"
+require "securerandom"
 require "yaml"
 
 module Mihari
@@ -12,24 +13,16 @@ module Mihari
       # @return [Hash]
       attr_reader :data
 
-      # @return [String]
-      attr_reader :yaml
-
       # @return [Array, nil]
       attr_reader :errors
 
-      # @return [String]
-      attr_writer :id
-
       #
       # Initialize
       #
       # @param [Hash] data
-      # @param [String] yaml
       #
-      def initialize(data, yaml)
+      def initialize(data)
         @data = data.deep_symbolize_keys
-        @yaml = yaml
 
         @errors = nil
 
@@ -70,7 +63,7 @@ module Mihari
       # @return [String]
       #
       def id
-        @id ||= data[:id] || UUIDTools::UUID.md5_create(UUIDTools::UUID_URL_NAMESPACE, data.to_yaml).to_s
+        @id ||= data[:id]
       end
 
       #
@@ -87,16 +80,71 @@ module Mihari
         @description ||= data[:description]
       end
 
+      #
+      # @return [String]
+      #
+      def yaml
+        @yaml ||= data.deep_stringify_keys.to_yaml
+      end
+
+      #
+      # @return [Array<Hash>]
+      #
+      def queries
+        @queries ||= data[:queries]
+      end
+
+      #
+      # @return [Array<String>]
+      #
+      def data_types
+        @data_types ||= data[:data_types]
+      end
+
+      #
+      # @return [Array<String>]
+      #
+      def tags
+        @tags ||= data[:tags]
+      end
+
+      #
+      # @return [Array<String>]
+      #
+      def falsepositives
+        @falsepositives ||= data[:falsepositives]
+      end
+
+      #
+      # @return [Array<Hash>]
+      #
+      def emitters
+        @emitters ||= data[:emitters]
+      end
+
+      #
+      # @return [Array<Hash>]
+      #
+      def enrichers
+        @enrichers ||= data[:enrichers]
+      end
+
+      #
+      # @return [Integer, nil]
+      #
+      def artifact_lifetime
+        @artifact_lifetime ||= data[:artifact_lifetime]
+      end
+
       #
       # @return [Mihari::Rule]
       #
-      def to_model
+      def model
         rule = Mihari::Rule.find(id)
 
         rule.title = title
         rule.description = description
         rule.data = data
-        rule.yaml = yaml
 
         rule
       rescue ActiveRecord::RecordNotFound
@@ -104,118 +152,83 @@ module Mihari
           id: id,
           title: title,
           description: description,
-          data: data,
-          yaml: yaml
+          data: data
         )
       end
 
       #
       # @return [Mihari::Analyzers::Rule]
       #
-      def to_analyzer
-        analyzer = Mihari::Analyzers::Rule.new(
-          title: self[:title],
-          description: self[:description],
-          tags: self[:tags],
-          queries: self[:queries],
-          allowed_data_types: self[:allowed_data_types],
-          disallowed_data_values: self[:disallowed_data_values],
-          emitters: self[:emitters],
-          enrichers: self[:enrichers],
-          id: id
-        )
-        analyzer.ignore_old_artifacts = self[:ignore_old_artifacts]
-        analyzer.ignore_threshold = self[:ignore_threshold]
-
-        analyzer
+      def analyzer
+        Mihari::Analyzers::Rule.new(rule: self)
       end
 
       class << self
         include Mixins::Database
 
         #
-        # @param [Mihari::Rule] model
-        #
-        # @return [Mihari::Structs::Rule]
-        #
-        def from_model(model)
-          data = model.data.deep_symbolize_keys
-          # set ID if YAML data do not have ID
-          data[:id] = model.id unless data.key?(:id)
-
-          Structs::Rule.new(data, model.yaml)
-        end
-
+        # Load rule from YAML string
         #
         # @param [String] yaml
         #
         # @return [Mihari::Structs::Rule]
-        # @param [String, nil] id
         #
-        def from_yaml(yaml, id: nil)
-          data = load_erb_yaml(yaml)
-          # set ID if id is given & YAML data do not have ID
-          data[:id] = id if !id.nil? && !data.key?(:id)
-
-          Structs::Rule.new(data, yaml)
+        def from_yaml(yaml)
+          Structs::Rule.new YAML.safe_load(ERB.new(yaml).result, permitted_classes: [Date, Symbol])
+        rescue Psych::SyntaxError => e
+          raise YAMLSyntaxError, e.message
         end
 
         #
-        # @param [String] path_or_id Path to YAML file or YAML string or ID of a rule in the database
+        # @param [Mihari::Rule] model
         #
         # @return [Mihari::Structs::Rule]
         #
-        def from_path_or_id(path_or_id)
-          yaml = nil
-
-          yaml = load_yaml_from_file(path_or_id) if File.exist?(path_or_id)
-          yaml = load_yaml_from_db(path_or_id) if yaml.nil?
-
-          Structs::Rule.from_yaml yaml
+        def from_model(model)
+          Structs::Rule.new(model.data)
         end
 
-        private
-
         #
-        # Load ERR templated YAML
+        # Load a rule from path
         #
-        # @param [String] yaml
+        # @param [String] path
         #
-        # @return [Hash]
+        # @return [Mihari::Structs::Rule, nil]
         #
-        def load_erb_yaml(yaml)
-          YAML.safe_load(ERB.new(yaml).result, permitted_classes: [Date, Symbol], symbolize_names: true)
-        rescue Psych::SyntaxError => e
-          raise YAMLSyntaxError, e.message
+        def from_path(path)
+          return nil unless Pathname(path).exist?
+
+          from_yaml File.read(path)
         end
 
         #
-        # Load YAML string from path
+        # Load a rule from DB
         #
-        # @param [String] path
+        # @param [String] id
         #
-        # @return [String, nil]
+        # @return [Mihari::Structs::Rule, nil]
         #
-        def load_yaml_from_file(path)
-          return nil unless Pathname(path).exist?
+        def from_id(id)
+          with_db_connection do
+            return nil unless Mihari::Rule.exists?(id)
 
-          File.read path
+            Structs::Rule.from_model Mihari::Rule.find(id)
+          end
         end
 
         #
-        # Load YAML string from the database
-        #
-        # @param [String] id <description>
+        # @param [String] path_or_id Path to YAML file or YAML string or ID of a rule in the database
         #
-        # @return [Hash]
+        # @return [Mihari::Structs::Rule]
         #
-        def load_yaml_from_db(id)
-          with_db_connection do
-            rule = Mihari::Rule.find(id)
-            rule.yaml || rule.symbolized_data.to_yaml
-          rescue ActiveRecord::RecordNotFound
-            raise ArgumentError, "ID:#{id} is not found in the database"
-          end
+        def from_path_or_id(path_or_id)
+          rule = from_path(path_or_id)
+          return rule unless rule.nil?
+
+          rule = from_id(path_or_id)
+          return rule unless rule.nil?
+
+          raise ArgumentError, "#{path_or_id} does not exist"
         end
       end
     end

data/lib/mihari/templates/rule.yml.erb

@@ -1,23 +1,5 @@
-title: ... # String (required)
-description: ... # String (required)
-
-id: ... # String (optional)
-author: ... # String (optional)
-created_on: <%= Date.today %> # Date (optional)
-updated_on: <%= Date.today %> # Date (optional)
-
-tags: [] # Array<String> (Optional, defaults to [])
-allowed_data_types: # Array<String> (Optional, defaults to ["hash", "ip", "domain", "url", "mail"])
-  - hash
-  - ip
-  - domain
-  - url
-  - mail
-disallowed_data_values: [] # Array<String> (Optional, defaults to [])
-
-ignore_old_artifacts: true # Whether to ignore old artifacts from checking or not (Optional, defaults to true)
-ignore_threshold: 0 # Number of days to define whether an artifact is old or not (Optional, defaults to 0)
-
-queries: # Array<Hash> (required)
-  - analyzer: shodan # String (required)
-    query: ... # String (required)
+id: <%= SecureRandom.uuid %>
+title: Title goes here
+description: Description goes here
+created_on: <%= Date.today %>
+queries: []

data/lib/mihari/types.rb

@@ -12,7 +12,7 @@ module Mihari
     Double = Strict::Float | Strict::Integer
     DateTime = Strict::DateTime
 
-    DataTypes = Types::String.enum(*ALLOWED_DATA_TYPES)
+    DataTypes = Types::String.enum(*DEFAULT_DATA_TYPES)
 
     HTTPRequestMethods = Types::String.enum("GET", "POST")
     HTTPRequestPayloadTypes = Types::String.enum("application/json", "application/x-www-form-urlencoded")

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "4.12.0"
+  VERSION = "5.0.0"
 end

data/lib/mihari/web/api.rb

@@ -6,7 +6,6 @@ require "mihari/web/endpoints/artifacts"
 require "mihari/web/endpoints/configs"
 require "mihari/web/endpoints/ip_addresses"
 require "mihari/web/endpoints/rules"
-require "mihari/web/endpoints/sources"
 require "mihari/web/endpoints/tags"
 
 module Mihari
@@ -19,7 +18,6 @@ module Mihari
     mount Endpoints::Configs
     mount Endpoints::IPAddresses
     mount Endpoints::Rules
-    mount Endpoints::Sources
     mount Endpoints::Tags
 
     add_swagger_documentation(api_version: "v1", info: { title: "Mihari API" })

data/lib/mihari/web/endpoints/alerts.rb

@@ -6,7 +6,7 @@ module Mihari
       namespace :alerts do
         desc "Search alerts", {
           is_array: true,
-          success: Entities::Alert,
+          success: Entities::AlertsWithPagination,
           failure: [{ code: 404, message: "Not found", model: Entities::Message }],
           summary: "Search alerts"
         }
@@ -15,7 +15,7 @@ module Mihari
 
           optional :artifact, type: String
           optional :description, type: String
-          optional :source, type: String
+          optional :rule_id, type: String
           optional :tag, type: String
           optional :title, type: String
 
@@ -47,7 +47,15 @@ module Mihari
           alerts = Mihari::Alert.search(search_filter_with_pagenation)
           total = Mihari::Alert.count(search_filter_with_pagenation.without_pagination)
 
-          present({ alerts: alerts, total: total, current_page: page, page_size: limit }, with: Entities::AlertsWithPagination)
+          present(
+            {
+              alerts: alerts,
+              total: total,
+              current_page: page,
+              page_size: limit
+            },
+            with: Entities::AlertsWithPagination
+          )
         end
 
         desc "Delete an alert", {

data/lib/mihari/web/endpoints/configs.rb

@@ -10,12 +10,7 @@ module Mihari
           summary: "Get configs"
         }
         get "/" do
-          statuses = Status.check
-
-          configs = statuses.map do |key, value|
-            { name: key, status: value }
-          end
-          present(configs, with: Entities::Config)
+          present(Mihari.configs, with: Entities::Config)
         end
       end
     end