mihari 4.11.0 → 5.0.0

data/lib/mihari/database.rb

@@ -17,53 +17,44 @@ def development_env?
   env == "development"
 end
 
-class InitialSchema < ActiveRecord::Migration[7.0]
+class V5Schema < ActiveRecord::Migration[7.0]
   def change
-    create_table :tags, if_not_exists: true do |t|
-      t.string :name, null: false
+    create_table :rules, id: :string, if_not_exists: true do |t|
+      t.string :title, null: false
+      t.string :description, null: false
+      t.json :data, null: false
+      t.timestamps
     end
 
     create_table :alerts, if_not_exists: true do |t|
-      t.string :title, null: false
-      t.string :description, null: true
-      t.string :source, null: false
       t.timestamps
+
+      t.belongs_to :rule, foreign_key: true, type: :string, null: false
     end
 
     create_table :artifacts, if_not_exists: true do |t|
       t.string :data, null: false
       t.string :data_type, null: false
-      t.belongs_to :alert, foreign_key: true
+      t.string :source
+      t.json :metadata
       t.timestamps
-    end
 
-    create_table :taggings, if_not_exists: true do |t|
-      t.integer :tag_id
-      t.integer :alert_id
+      t.belongs_to :alert, foreign_key: true, null: false
     end
 
-    add_index :taggings, :tag_id, if_not_exists: true
-    add_index :taggings, [:tag_id, :alert_id], unique: true, if_not_exists: true
-  end
-end
-
-class AddeSourceToArtifactSchema < ActiveRecord::Migration[7.0]
-  def change
-    add_column :artifacts, :source, :string, if_not_exists: true
-  end
-end
-
-class EnrichmentsSchema < ActiveRecord::Migration[7.0]
-  def change
     create_table :autonomous_systems, if_not_exists: true do |t|
       t.integer :asn, null: false
-      t.belongs_to :artifact, foreign_key: true
+      t.datetime :created_at
+
+      t.belongs_to :artifact, foreign_key: true, null: false
     end
 
     create_table :geolocations, if_not_exists: true do |t|
       t.string :country, null: false
       t.string :country_code, null: false
-      t.belongs_to :artifact, foreign_key: true
+      t.datetime :created_at
+
+      t.belongs_to :artifact, foreign_key: true, null: false
     end
 
     create_table :whois_records, if_not_exists: true do |t|
@@ -73,73 +64,59 @@ class EnrichmentsSchema < ActiveRecord::Migration[7.0]
       t.date :expires_on
       t.json :registrar
       t.json :contacts
-      t.belongs_to :artifact, foreign_key: true
+      t.datetime :created_at
+
+      t.belongs_to :artifact, foreign_key: true, null: false
     end
 
     create_table :dns_records, if_not_exists: true do |t|
       t.string :resource, null: false
       t.string :value, null: false
-      t.belongs_to :artifact, foreign_key: true
+      t.datetime :created_at
+
+      t.belongs_to :artifact, foreign_key: true, null: false
     end
 
     create_table :reverse_dns_names, if_not_exists: true do |t|
       t.string :name, null: false
-      t.belongs_to :artifact, foreign_key: true
-    end
-  end
-end
+      t.datetime :created_at
 
-class EnrichmentCreatedAtSchema < ActiveRecord::Migration[7.0]
-  def change
-    # Add created_at column because now it is able to enrich an atrifact after the creation
-    add_column :autonomous_systems, :created_at, :datetime, if_not_exists: true
-    add_column :geolocations, :created_at, :datetime, if_not_exists: true
-    add_column :whois_records, :created_at, :datetime, if_not_exists: true
-    add_column :dns_records, :created_at, :datetime, if_not_exists: true
-    add_column :reverse_dns_names, :created_at, :datetime, if_not_exists: true
-  end
-end
-
-class RuleSchema < ActiveRecord::Migration[7.0]
-  def change
-    create_table :rules, id: :string, if_not_exists: true do |t|
-      t.string :title, null: false
-      t.string :description, null: false
-      t.json :data, null: false
-      t.timestamps
+      t.belongs_to :artifact, foreign_key: true, null: false
     end
-  end
-end
-
-class AddeMetadataToArtifactSchema < ActiveRecord::Migration[7.0]
-  def change
-    add_column :artifacts, :metadata, :json, if_not_exists: true
-  end
-end
 
-class AddYAMLToRulesSchema < ActiveRecord::Migration[7.0]
-  def change
-    add_column :rules, :yaml, :text, if_not_exists: true
-  end
-end
-
-class EnrichmentsV45Schema < ActiveRecord::Migration[7.0]
-  def change
     create_table :cpes, if_not_exists: true do |t|
       t.string :cpe, null: false
-      t.belongs_to :artifact, foreign_key: true
+      t.datetime :created_at
+
+      t.belongs_to :artifact, foreign_key: true, null: false
     end
 
     create_table :ports, if_not_exists: true do |t|
       t.integer :port, null: false
-      t.belongs_to :artifact, foreign_key: true
+      t.datetime :created_at
+
+      t.belongs_to :artifact, foreign_key: true, null: false
+    end
+
+    create_table :tags, if_not_exists: true do |t|
+      t.string :name, null: false
+      t.datetime :created_at
     end
+
+    create_table :taggings, if_not_exists: true do |t|
+      t.integer :tag_id
+      t.integer :alert_id
+      t.datetime :created_at
+    end
+
+    add_index :taggings, :tag_id, if_not_exists: true
+    add_index :taggings, %i[tag_id alert_id], unique: true, if_not_exists: true
   end
 end
 
 def adapter
-  return "postgresql" if Mihari.config.database.start_with?("postgresql://", "postgres://")
-  return "mysql2" if Mihari.config.database.start_with?("mysql2://")
+  return "postgresql" if %w[postgresql postgres].include?(Mihari.config.database_url.scheme)
+  return "mysql2" if Mihari.config.database_url.scheme == "mysql2"
 
   "sqlite3"
 end
@@ -157,19 +134,7 @@ module Mihari
       def migrate(direction)
         ActiveRecord::Migration.verbose = false
 
-        [
-          InitialSchema,
-          AddeSourceToArtifactSchema,
-          EnrichmentsSchema,
-          EnrichmentCreatedAtSchema,
-          # v4.0
-          RuleSchema,
-          AddeMetadataToArtifactSchema,
-          # v4.4
-          AddYAMLToRulesSchema,
-          # v4.5
-          EnrichmentsV45Schema
-        ].each { |schema| schema.migrate direction }
+        [V5Schema].each { |schema| schema.migrate direction }
       end
       memoize :migrate unless test_env?
 
@@ -181,19 +146,19 @@ module Mihari
 
         case adapter
         when "postgresql", "mysql2"
-          ActiveRecord::Base.establish_connection(Mihari.config.database)
+          ActiveRecord::Base.establish_connection(Mihari.config.database_url.to_s)
         else
           ActiveRecord::Base.establish_connection(
             adapter: adapter,
-            database: Mihari.config.database
+            database: Mihari.config.database_url.path[1..]
           )
         end
 
         ActiveRecord::Base.logger = Logger.new($stdout) if development_env?
 
         migrate :up
-      rescue StandardError
-        # Do nothing
+      rescue StandardError => e
+        Mihari.logger.error e
       end
 
       #

data/lib/mihari/emitters/database.rb

@@ -7,23 +7,32 @@ module Mihari
         true
       end
 
-      def emit(title:, description:, artifacts:, source:, tags: [])
+      #
+      # Create an alert
+      #
+      # @param [Arra<Mihari::Artifact>] artifacts
+      # @param [Mihari::Structs::Rule] rule
+      #
+      # @return [Mihari::Alert]
+      #
+      def emit(artifacts:, rule:)
         return if artifacts.empty?
 
-        tags = tags.filter_map { |name| Tag.find_or_create_by(name: name) }.uniq
+        tags = rule.tags.filter_map { |name| Tag.find_or_create_by(name: name) }.uniq
         taggings = tags.map { |tag| Tagging.new(tag_id: tag.id) }
 
         alert = Alert.new(
-          title: title,
-          description: description,
           artifacts: artifacts,
-          source: source,
-          taggings: taggings
+          taggings: taggings,
+          rule_id: rule.id
         )
-
         alert.save
         alert
       end
+
+      def configuration_keys
+        %w[database_url]
+      end
     end
   end
 end

data/lib/mihari/emitters/misp.rb

@@ -14,7 +14,7 @@ module Mihari
       def initialize(*args, **kwargs)
         super(*args, **kwargs)
 
-        @url = kwargs[:url] || kwargs[:api_endpoint] || Mihari.config.misp_url
+        @url = kwargs[:url] || Mihari.config.misp_url
         @api_key = kwargs[:api_key] || Mihari.config.misp_api_key
 
         ::MISP.configure do |config|
@@ -39,16 +39,24 @@ module Mihari
         true
       end
 
-      def emit(title:, artifacts:, tags: [], **_options)
+      #
+      # Create a MISP event
+      #
+      # @param [Arra<Mihari::Artifact>] artifacts
+      # @param [Mihari::Structs::Rule] rule
+      #
+      # @return [::MISP::Event]
+      #
+      def emit(rule:, artifacts:, **_options)
         return if artifacts.empty?
 
-        event = ::MISP::Event.new(info: title)
+        event = ::MISP::Event.new(info: rule.title)
 
         artifacts.each do |artifact|
           event.attributes << build_attribute(artifact)
         end
 
-        tags.each do |tag|
+        rule.tags.each do |tag|
           event.add_tag name: tag
         end
 
@@ -66,7 +74,7 @@ module Mihari
       #
       # @param [Mihari::Artifact] artifact
       #
-      # @return [::MISP::Attribute] <description>
+      # @return [::MISP::Attribute]
       #
       def build_attribute(artifact)
         ::MISP::Attribute.new(value: artifact.data, type: to_misp_type(type: artifact.data_type, value: artifact.data))

data/lib/mihari/emitters/slack.rb

@@ -167,30 +167,37 @@ module Mihari
       #
       # Build a text
       #
-      # @param [String] title
-      # @param [String] description
-      # @param [Array<String>] tags
+      # @param [Mihari::Structs::Rule] rule
       #
       # @return [String]
       #
-      def to_text(title:, description:, tags: [])
+      def to_text(rule)
+        tags = rule.tags
         tags = ["N/A"] if tags.empty?
 
         [
-          "*#{title}*",
-          "*Desc.*: #{description}",
+          "*#{rule.title}*",
+          "*Desc.*: #{rule.description}",
           "*Tags*: #{tags.join(", ")}"
         ].join("\n")
       end
 
-      def emit(title:, description:, artifacts:, tags: [], **_options)
+      def emit(rule:, artifacts:, **_options)
         return if artifacts.empty?
 
         attachments = to_attachments(artifacts)
-        text = to_text(title: title, description: description, tags: tags)
+        text = to_text(rule)
 
         notifier.post(text: text, attachments: attachments, mrkdwn: true)
       end
+
+      def configuration_keys
+        %w[slack_webhook_url slack_channel]
+      end
+
+      def configured?
+        valid?
+      end
     end
   end
 end

data/lib/mihari/emitters/the_hive.rb

@@ -17,7 +17,7 @@ module Mihari
       def initialize(*args, **kwargs)
         super(*args, **kwargs)
 
-        @url = kwargs[:url] || kwargs[:api_endpoint] || Mihari.config.thehive_url
+        @url = kwargs[:url] || Mihari.config.thehive_url
         @api_key = kwargs[:api_key] || Mihari.config.thehive_api_key
         @api_version = kwargs[:api_version] || Mihari.config.thehive_api_version
       end
@@ -38,10 +38,18 @@ module Mihari
         true
       end
 
-      def emit(title:, description:, artifacts:, tags: [], **_options)
+      #
+      # Create a Hive alert
+      #
+      # @param [Arra<Mihari::Artifact>] artifacts
+      # @param [Mihari::Structs::Rule] rule
+      #
+      # @return [::MISP::Event]
+      #
+      def emit(rule:, artifacts:, **_options)
         return if artifacts.empty?
 
-        payload = payload(title: title, description: description, artifacts: artifacts, tags: tags)
+        payload = payload(rule: rule, artifacts: artifacts)
         api.alert.create(**payload)
       end
 
@@ -93,36 +101,49 @@ module Mihari
         !api_key.nil?
       end
 
-      def payload(title:, description:, artifacts:, tags: [])
-        if normalized_api_version.nil?
-          return v4_payload(title: title, description: description, artifacts: artifacts,
-            tags: tags)
-        end
+      #
+      # Build payload for alert
+      #
+      # @param [Arra<Mihari::Artifact>] artifacts
+      # @param [Mihari::Structs::Rule] rule
+      #
+      # @return [<Type>] <description>
+      #
+      def payload(rule:, artifacts:)
+        return v4_payload(rule: rule, artifacts: artifacts) if normalized_api_version.nil?
 
-        v5_payload(title: title, description: description, artifacts: artifacts, tags: tags)
+        v5_payload(rule: rule, artifacts: artifacts)
       end
 
-      def v4_payload(title:, description:, artifacts:, tags: [])
+      def v4_payload(rule:, artifacts:)
         {
-          title: title,
-          description: description,
+          title: rule.title,
+          description: rule.description,
           artifacts: artifacts.map do |artifact|
-                       { data: artifact.data, data_type: artifact.data_type, message: description }
-                     end,
-          tags: tags,
+            {
+              data: artifact.data,
+              data_type: artifact.data_type,
+              message: rule.description
+            }
+          end,
+          tags: rule.tags,
           type: "external",
           source: "mihari"
         }
       end
 
-      def v5_payload(title:, description:, artifacts:, tags: [])
+      def v5_payload(rule:, artifacts:)
         {
-          title: title,
-          description: description,
+          title: rule.title,
+          description: rule.description,
           observables: artifacts.map do |artifact|
-                         { data: artifact.data, data_type: artifact.data_type, message: description }
-                       end,
-          tags: tags,
+            {
+              data: artifact.data,
+              data_type: artifact.data_type,
+              message: rule.description
+            }
+          end,
+          tags: rule.tags,
           type: "external",
           source: "mihari",
           source_ref: "1"

data/lib/mihari/emitters/webhook.rb

@@ -1,54 +1,122 @@
 # frozen_string_literal: true
 
+require "erb"
+
 module Mihari
   module Emitters
-    class Webhook < Base
-      # @return [Boolean]
-      def valid?
-        webhook_url?
+    class PayloadTemplate < ERB
+      def self.template
+        %{
+					{
+						"rule": {
+              "id": "<%= @rule.id %>",
+              "title": "<%= @rule.title %>",
+              "description": "<%= @rule.description %>"
+            },
+						"artifacts": [
+							<% @artifacts.each_with_index do |artifact, idx| %>
+								"<%= artifact.data %>"
+								<%= ',' if idx < (@artifacts.length - 1) %>
+							<% end %>
+						],
+						"tags": [
+							<% @rule.tags.each_with_index do |tag, idx| %>
+								"<%= tag %>"
+								<%= ',' if idx < (@rule.tags.length - 1) %>
+							<% end %>
+						]
+					}
+				}
       end
 
-      def emit(title:, description:, artifacts:, source:, tags:)
-        return if artifacts.empty?
+      def initialize(artifacts:, rule:, options: {})
+        @artifacts = artifacts
+        @rule = rule
 
-        headers = { "content-type": "application/x-www-form-urlencoded" }
-        headers["content-type"] = "application/json" if use_json_body?
+        @template = options.fetch(:template, self.class.template)
+        super(@template)
+      end
 
-        emitter = Emitters::HTTP.new(uri: Mihari.config.webhook_url)
-        emitter.emit(title: title, description: description, artifacts: artifacts, source: source, tags: tags)
+      def result
+        super(binding)
       end
+    end
 
-      private
+    class Webhook < Base
+      # @return [Addressable::URI, nil]
+      attr_reader :url
 
-      def configuration_keys
-        %w[webhook_url]
-      end
+      # @return [Hash]
+      attr_reader :headers
+
+      # @return [String]
+      attr_reader :method
 
-      #
-      # Webhook URL
-      #
       # @return [String, nil]
-      #
-      def webhook_url
-        @webhook_url ||= Mihari.config.webhook_url
+      attr_reader :template
+
+      def initialize(*args, **kwargs)
+        super(*args, **kwargs)
+
+        url = kwargs[:url]
+        headers = kwargs[:headers] || {}
+        method = kwargs[:method] || "POST"
+        template = kwargs[:template]
+
+        @url = Addressable::URI.parse(url)
+        @headers = headers
+        @method = method
+        @template = template
       end
 
-      #
-      # Check whether a webhook URL is set or not
-      #
-      # @return [Boolean]
-      #
-      def webhook_url?
-        !webhook_url.nil?
+      def emit(artifacts:, rule:)
+        return if artifacts.empty?
+
+        res = nil
+
+        payload_ = payload_as_string(artifacts: artifacts, rule: rule)
+        payload = JSON.parse(payload_)
+
+        client = Mihari::HTTP.new(url, headers: headers, payload: payload)
+
+        case method
+        when "GET"
+          res = client.get
+        when "POST"
+          res = client.post
+        end
+
+        res
       end
 
+      def valid?
+        return false if url.nil?
+
+        %w[http https].include? url.scheme.downcase
+      end
+
+      private
+
       #
-      # Check whether to use JSON body or not
+      # Convert payload into string
       #
-      # @return [Boolean]
+      # @param [Array<Mihari::Artifact>] artifacts
+      # @param [Mihari::Structs::Rule] rule
       #
-      def use_json_body?
-        @use_json_body ||= Mihari.config.webhook_use_json_body
+      # @return [String]
+      #
+      def payload_as_string(artifacts:, rule:)
+        @payload_as_string ||= [].tap do |out|
+          options = {}
+          options[:template] = File.read(template) unless template.nil?
+
+          payload_template = PayloadTemplate.new(
+            artifacts: artifacts,
+            rule: rule,
+            options: options
+          )
+          out << payload_template.result
+        end.first
       end
     end
   end

data/lib/mihari/entities/alert.rb

@@ -3,13 +3,15 @@
 module Mihari
   module Entities
     class Alert < Grape::Entity
-      expose :id, documentation: { type: Integer, required: true }
-      expose :title, documentation: { type: String, required: true }
-      expose :description, documentation: { type: String, required: true }
-      expose :source, documentation: { type: String, required: true }
+      expose :id,
+        documentation: { type: String, required: true,
+                         desc: "String representation of the ID" } do |alert, _options|
+        alert.id.to_s
+      end
+      expose :rule_id, documentation: { type: String, required: true }, as: :ruleId
       expose :created_at, documentation: { type: DateTime, required: true }, as: :createdAt
 
-      expose :artifacts, using: Entities::Artifact, documentation: { type: Entities::Artifact, is_array: true }
+      expose :artifacts, using: Entities::BaseArtifact, documentation: { type: Entities::BaseArtifact, is_array: true }
       expose :tags, using: Entities::Tag, documentation: { type: Entities::Tag, is_array: true, required: true }
     end
 

data/lib/mihari/entities/artifact.rb

@@ -2,29 +2,41 @@
 
 module Mihari
   module Entities
-    class Artifact < Grape::Entity
-      expose :id, documentation: { type: Integer, required: true }
+    class BaseArtifact < Grape::Entity
+      expose :id,
+        documentation: { type: String, required: true,
+                         desc: "String representation of the ID" } do |artifact, _options|
+        artifact.id.to_s
+      end
       expose :data, documentation: { type: String, required: true }
       expose :data_type, documentation: { type: String, required: true }, as: :dataType
       expose :source, documentation: { type: String, required: true }
       expose :tags, documentation: { type: String, is_array: true }
 
       expose :metadata, documentation: { type: Hash }
+    end
 
-      expose :autonomous_system, using: Entities::AutonomousSystem, documentation: { type: Entities::AutonomousSystem, required: false }, as: :autonomousSystem
+    class Artifact < BaseArtifact
+      expose :autonomous_system, using: Entities::AutonomousSystem,
+        documentation: { type: Entities::AutonomousSystem, required: false }, as: :autonomousSystem
       expose :geolocation, using: Entities::Geolocation, documentation: { type: Entities::Geolocation, required: false }
-      expose :whois_record, using: Entities::WhoisRecord, documentation: { type: Entities::WhoisRecord, required: false }, as: :whoisRecord
+      expose :whois_record, using: Entities::WhoisRecord,
+        documentation: { type: Entities::WhoisRecord, required: false }, as: :whoisRecord
 
-      expose :reverse_dns_names, using: Entities::ReverseDnsName, documentation: { type: Entities::ReverseDnsName, is_array: true, required: false }, as: :reverseDnsNames do |status, _options|
+      expose :reverse_dns_names, using: Entities::ReverseDnsName,
+        documentation: { type: Entities::ReverseDnsName, is_array: true, required: false }, as: :reverseDnsNames do |status, _options|
         status.reverse_dns_names.empty? ? nil : status.reverse_dns_names
       end
-      expose :dns_records, using: Entities::DnsRecord, documentation: { type: Entities::DnsRecord, is_array: true, required: false }, as: :dnsRecords do |status, _options|
+      expose :dns_records, using: Entities::DnsRecord,
+        documentation: { type: Entities::DnsRecord, is_array: true, required: false }, as: :dnsRecords do |status, _options|
         status.dns_records.empty? ? nil : status.dns_records
       end
-      expose :ceps, using: Entities::CPE, documentation: { type: Entities::CPE, is_array: true, required: false }, as: :cpes do |status, _options|
+      expose :ceps, using: Entities::CPE, documentation: { type: Entities::CPE, is_array: true, required: false },
+        as: :cpes do |status, _options|
         status.cpes.empty? ? nil : status.cpes
       end
-      expose :ports, using: Entities::Port, documentation: { type: Entities::Port, is_array: true, required: false }, as: :ports do |status, _options|
+      expose :ports, using: Entities::Port, documentation: { type: Entities::Port, is_array: true, required: false },
+        as: :ports do |status, _options|
         status.ports.empty? ? nil : status.ports
       end
     end