mihari 6.3.0 → 7.0.0

data/lib/mihari/emitters/misp.rb

@@ -56,8 +56,10 @@ module Mihari
         })
       end
 
-      def configuration_keys
-        %w[misp_url misp_api_key]
+      class << self
+        def configuration_keys
+          %w[misp_url misp_api_key]
+        end
       end
 
       private
@@ -126,15 +128,6 @@ module Mihari
       def url?
         !url.nil? && !url.empty?
       end
-
-      #
-      # Check whether an API key is set or not
-      #
-      # @return [Boolean]
-      #
-      def api_key?
-        !api_key.nil? && !api_key.empty?
-      end
     end
   end
 end

data/lib/mihari/emitters/slack.rb

@@ -201,7 +201,7 @@ module Mihari
       # @return [String]
       #
       def text
-        tags = rule.tags
+        tags = rule.tags.map(&:name)
         tags = ["N/A"] if tags.empty?
         [
           "*#{rule.title}*",
@@ -214,15 +214,17 @@ module Mihari
       # @param [Array<Mihari::Models::Artifact>] artifacts
       #
       def call(artifacts)
-        @artifacts = artifacts
-
         return if artifacts.empty?
 
+        @artifacts = artifacts
+
         notifier.post(text: text, attachments: attachments, mrkdwn: true)
       end
 
-      def configuration_keys
-        %w[slack_webhook_url slack_channel]
+      class << self
+        def configuration_keys
+          %w[slack_webhook_url slack_channel]
+        end
       end
     end
   end

data/lib/mihari/emitters/the_hive.rb

@@ -9,9 +9,6 @@ module Mihari
       # @return [String, nil]
       attr_reader :api_key
 
-      # @return [String, nil]
-      attr_reader :api_version
-
       # @return [Array<Mihari::Models::Artifact>]
       attr_accessor :artifacts
 
@@ -25,7 +22,6 @@ module Mihari
 
         @url = params[:url] || Mihari.config.thehive_url
         @api_key = params[:api_key] || Mihari.config.thehive_api_key
-        @api_version = params[:api_version] || Mihari.config.thehive_api_version
 
         @artifacts = []
       end
@@ -43,37 +39,23 @@ module Mihari
       # @param [Array<Mihari::Models::Artifact>] artifacts
       #
       def call(artifacts)
-        @artifacts = artifacts
-
         return if artifacts.empty?
 
-        client.alert payload
-      end
+        @artifacts = artifacts
 
-      #
-      # Normalize API version for API client
-      #
-      # @param [String] version
-      #
-      # @return [String, nil]
-      #
-      def normalized_api_version
-        @normalized_api_version ||= [].tap do |out|
-          # v4 does not have version prefix in path (/api/)
-          # v5 has version prefix in path (/api/v1/)
-          table = { "" => nil, "v4" => nil, "v5" => "v1" }
-          out << table[api_version.to_s.downcase]
-        end.first
+        client.alert payload
       end
 
-      def configuration_keys
-        %w[thehive_url thehive_api_key]
+      class << self
+        def configuration_keys
+          %w[thehive_url thehive_api_key]
+        end
       end
 
       private
 
       def client
-        @client ||= Clients::TheHive.new(url, api_key: api_key, api_version: normalized_api_version, timeout: timeout)
+        Clients::TheHive.new(url, api_key: api_key, api_version: "v1", timeout: timeout)
       end
 
       #
@@ -85,44 +67,12 @@ module Mihari
         !url.nil?
       end
 
-      #
-      # Check whether an API key is set or not
-      #
-      # @return [Boolean]
-      #
-      def api_key?
-        !api_key.nil?
-      end
-
       #
       # Build payload for alert
       #
       # @return [Hash]
       #
       def payload
-        return v4_payload if normalized_api_version.nil?
-
-        v5_payload
-      end
-
-      def v4_payload
-        {
-          title: rule.title,
-          description: rule.description,
-          artifacts: artifacts.map do |artifact|
-            {
-              data: artifact.data,
-              data_type: artifact.data_type,
-              message: rule.description
-            }
-          end,
-          tags: rule.tags,
-          type: "external",
-          source: "mihari"
-        }
-      end
-
-      def v5_payload
         {
           title: rule.title,
           description: rule.description,
@@ -136,7 +86,7 @@ module Mihari
           tags: rule.tags,
           type: "external",
           source: "mihari",
-          source_ref: "1"
+          source_ref: SecureRandom.uuid
         }
       end
     end

data/lib/mihari/emitters/webhook.rb

@@ -4,7 +4,7 @@ require "erb"
 
 module Mihari
   module Emitters
-    class PayloadTemplate < ERB
+    class ERBTemplate < ERB
       class << self
         def template
           %{
@@ -22,7 +22,7 @@ module Mihari
               ],
               "tags": [
                 <% @rule.tags.each_with_index do |tag, idx| %>
-                  "<%= tag %>"
+                  "<%= tag.name %>"
                   <%= ',' if idx < (@rule.tags.length - 1) %>
                 <% end %>
               ]
@@ -113,23 +113,23 @@ module Mihari
       #
       # @return [String]
       #
-      def rendered_template
+      def render
         options = {}
         options[:template] = File.read(template) unless template.nil?
 
-        payload_template = PayloadTemplate.new(
+        erb_template = ERBTemplate.new(
           artifacts: artifacts,
           rule: rule,
           options: options
         )
-        payload_template.result
+        erb_template.result
       end
 
       #
       # @return [Hash]
       #
       def json
-        JSON.parse rendered_template
+        JSON.parse render
       end
     end
   end

data/lib/mihari/enrichers/google_public_dns.rb

@@ -29,7 +29,7 @@ module Mihari
       private
 
       def client
-        Clients::GooglePublicDNS.new
+        Clients::GooglePublicDNS.new(timeout: timeout)
       end
     end
   end

data/lib/mihari/enrichers/mmdb.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Enrichers
+    #
+    # MMDB enricher
+    #
+    class MMDB < Base
+      #
+      # Query MMDB
+      #
+      # @param [String] ip
+      #
+      # @return [Mihari::Structs::MMDB::Response]
+      #
+      def call(ip)
+        client.query ip
+      end
+      memo_wise :call
+
+      private
+
+      def client
+        @client ||= Clients::MMDB.new(timeout: timeout)
+      end
+    end
+  end
+end

data/lib/mihari/enrichers/shodan.rb

@@ -14,16 +14,14 @@ module Mihari
       # @return [Mihari::Structs::Shodan::InternetDBResponse, nil]
       #
       def call(ip)
-        url = "https://internetdb.shodan.io/#{ip}"
-        res = http.get(url)
-        Structs::Shodan::InternetDBResponse.from_dynamic! JSON.parse(res.body.to_s)
+        client.query ip
       end
       memo_wise :call
 
       private
 
-      def http
-        HTTP::Factory.build timeout: timeout
+      def client
+        @client ||= Clients::ShodanInternetDB.new(timeout: timeout)
       end
     end
   end

data/lib/mihari/enrichers/whois.rb

@@ -23,7 +23,7 @@ module Mihari
       # @return [Mihari::Models::WhoisRecord, nil]
       #
       def call(domain)
-        _call PublicSuffix.domain(domain)
+        memoized_call PublicSuffix.domain(domain)
       end
 
       private
@@ -33,7 +33,7 @@ module Mihari
       #
       # @return [Mihari::Models::WhoisRecord, nil]
       #
-      def _call(domain)
+      def memoized_call(domain)
         record = whois.lookup(domain)
         parser = record.parser
         return nil if parser.available?
@@ -47,7 +47,7 @@ module Mihari
           contacts: get_contacts(parser)
         )
       end
-      memo_wise :_call
+      memo_wise :memoized_call
 
       #
       # @return [::Whois::Client]

data/lib/mihari/entities/alert.rb

@@ -3,11 +3,7 @@
 module Mihari
   module Entities
     class Alert < Grape::Entity
-      expose :id,
-        documentation: { type: String, required: true,
-                         desc: "String representation of the ID" } do |alert, _options|
-        alert.id.to_s
-      end
+      expose :id, documentation: { type: Integer, required: true }
       expose :rule_id, documentation: { type: String, required: true }, as: :ruleId
       expose :created_at, documentation: { type: DateTime, required: true }, as: :createdAt
 
@@ -15,11 +11,8 @@ module Mihari
       expose :tags, using: Entities::Tag, documentation: { type: Entities::Tag, is_array: true, required: true }
     end
 
-    class AlertsWithPagination < Grape::Entity
-      expose :alerts, using: Entities::Alert, documentation: { type: Entities::Alert, is_array: true, required: true }
-      expose :total, documentation: { type: Integer, required: true }
-      expose :current_page, documentation: { type: Integer, required: true }, as: :currentPage
-      expose :page_size, documentation: { type: Integer, required: true }, as: :pageSize
+    class AlertsWithPagination < Pagination
+      expose :results, using: Entities::Alert, documentation: { type: Entities::Alert, is_array: true, required: true }
     end
   end
 end

data/lib/mihari/entities/artifact.rb

@@ -3,22 +3,17 @@
 module Mihari
   module Entities
     class BaseArtifact < Grape::Entity
-      expose :id,
-        documentation: { type: String, required: true,
-                         desc: "String representation of the ID" } do |artifact, _options|
-        artifact.id.to_s
-      end
+      expose :id, documentation: { type: Integer, required: true }
       expose :data, documentation: { type: String, required: true }
       expose :data_type, documentation: { type: String, required: true }, as: :dataType
       expose :source, documentation: { type: String, required: true }
       expose :query, documentation: { type: String, required: false }
-      expose :tags, documentation: { type: String, is_array: true }
+      expose :metadata, documentation: { type: Hash }
+      expose :created_at, documentation: { type: DateTime, required: true }, as: :createdAt
     end
 
     class Artifact < BaseArtifact
-      # NOTE: do not define metadata in BaseArtifact since metadata can be relatively big
-      expose :metadata, documentation: { type: Hash }
-
+      expose :tags, using: Entities::Tag, documentation: { type: Entities::Tag, is_array: true, required: true }
       expose :autonomous_system, using: Entities::AutonomousSystem,
         documentation: { type: Entities::AutonomousSystem, required: false }, as: :autonomousSystem
       expose :geolocation, using: Entities::Geolocation, documentation: { type: Entities::Geolocation, required: false }
@@ -43,12 +38,9 @@ module Mihari
       end
     end
 
-    class ArtifactsWithPagination < Grape::Entity
-      expose :artifacts, using: Entities::Artifact,
+    class ArtifactsWithPagination < Pagination
+      expose :results, using: Entities::BaseArtifact,
         documentation: { type: Entities::Artifact, is_array: true, required: true }
-      expose :total, documentation: { type: Integer, required: true }
-      expose :current_page, documentation: { type: Integer, required: true }, as: :currentPage
-      expose :page_size, documentation: { type: Integer, required: true }, as: :pageSize
     end
   end
 end

data/lib/mihari/entities/config.rb

@@ -5,8 +5,8 @@ module Mihari
     class Config < Grape::Entity
       expose :name, documentation: { type: String, required: true }
       expose :type, documentation: { type: String, required: true }
-      expose :values, documentation: { type: String, is_array: true, required: true }
-      expose :is_configured, documentation: { type: Grape::API::Boolean, required: true }, as: :isConfigured
+      expose :items, documentation: { type: Hash, is_array: true, required: true }
+      expose :configured, documentation: { type: Grape::API::Boolean, required: true }
     end
   end
 end

data/lib/mihari/entities/cpe.rb

@@ -4,6 +4,7 @@ module Mihari
   module Entities
     class CPE < Grape::Entity
       expose :cpe, documentation: { type: String, required: true }
+      expose :created_at, documentation: { type: DateTime, required: true }, as: :createdAt
     end
   end
 end

data/lib/mihari/entities/dns.rb

@@ -5,6 +5,7 @@ module Mihari
     class DnsRecord < Grape::Entity
       expose :resource, documentation: { type: String, required: true }
       expose :value, documentation: { type: String, required: true }
+      expose :created_at, documentation: { type: DateTime, required: true }, as: :createdAt
     end
   end
 end

data/lib/mihari/entities/geolocation.rb

@@ -5,6 +5,7 @@ module Mihari
     class Geolocation < Grape::Entity
       expose :country, documentation: { type: String, required: true }
       expose :country_code, documentation: { type: String, required: true }, as: :countryCode
+      expose :created_at, documentation: { type: DateTime, required: true }, as: :createdAt
     end
   end
 end

data/lib/mihari/entities/ip_address.rb

@@ -3,11 +3,9 @@
 module Mihari
   module Entities
     class IPAddress < Grape::Entity
-      expose :ip, documentation: { type: String, required: true }
       expose :country_code, documentation: { type: String, required: true }, as: :countryCode
-      expose :hostname, documentation: { type: String, required: false }
-      expose :loc, documentation: { type: String, required: true }
       expose :asn, documentation: { type: Integer, required: false }
+      expose :loc, documentation: { type: String, required: false }
     end
   end
 end

data/lib/mihari/entities/messages.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Entities
+    class Message < Grape::Entity
+      expose :message, documentation: { type: String, required: true }
+    end
+
+    class ErrorMessage < Message
+      expose :detail, documentation: { type: Hash, required: false }
+    end
+
+    class QueueMessage < Message
+      expose :queued, documentation: { type: Grape::API::Boolean, required: true }
+    end
+  end
+end

data/lib/mihari/entities/pagination.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Entities
+    class Pagination < Grape::Entity
+      expose :total, documentation: { type: Integer, required: true }
+      expose :current_page, documentation: { type: Integer, required: true }, as: :currentPage
+      expose :page_size, documentation: { type: Integer, required: true }, as: :pageSize
+    end
+  end
+end

data/lib/mihari/entities/port.rb

@@ -4,6 +4,7 @@ module Mihari
   module Entities
     class Port < Grape::Entity
       expose :port, documentation: { type: Integer, required: true }
+      expose :created_at, documentation: { type: DateTime, required: true }, as: :createdAt
     end
   end
 end

data/lib/mihari/entities/reverse_dns.rb

@@ -4,6 +4,7 @@ module Mihari
   module Entities
     class ReverseDnsName < Grape::Entity
       expose :name, documentation: { type: String, required: true }
+      expose :created_at, documentation: { type: DateTime, required: true }, as: :createdAt
     end
   end
 end

data/lib/mihari/entities/rule.rb

@@ -2,16 +2,6 @@
 
 module Mihari
   module Entities
-    class Query < Grape::Entity
-      expose :analyzer, documentation: { type: String, required: true }
-      expose :query, documentation: { type: String, required: true }
-      expose :interval, documentation: { type: Integer, required: false }
-    end
-
-    class Emitter < Grape::Entity
-      expose :emitter, documentation: { type: String, required: true }
-    end
-
     class Rule < Grape::Entity
       expose :id, documentation: { type: String, required: true }
       expose :title, documentation: { type: String, required: true }
@@ -19,19 +9,11 @@ module Mihari
       expose :yaml, documentation: { type: String, required: true }
       expose :created_at, documentation: { type: DateTime, required: true }, as: :createdAt
       expose :updated_at, documentation: { type: DateTime, required: true }, as: :updatedAt
-
       expose :tags, using: Entities::Tag, documentation: { type: Entities::Tag, is_array: true, required: true }
     end
 
-    class RulesWithPagination < Grape::Entity
-      expose :rules, using: Entities::Rule, documentation: { type: Entities::Rule, is_array: true, required: true }
-      expose :total, documentation: { type: Integer, required: true }
-      expose :current_page, documentation: { type: Integer, required: true }, as: :currentPage
-      expose :page_size, documentation: { type: Integer, required: true }, as: :pageSize
-    end
-
-    class RuleIDs < Grape::Entity
-      expose :rule_ids, documentation: { type: [String], required: true }, as: :ruleIds
+    class RulesWithPagination < Pagination
+      expose :results, using: Entities::Rule, documentation: { type: Entities::Rule, is_array: true, required: true }
     end
   end
 end

data/lib/mihari/entities/tag.rb

@@ -7,8 +7,8 @@ module Mihari
       expose :name, documentation: { type: String, required: true }
     end
 
-    class Tags < Grape::Entity
-      expose :tags, documentation: { type: [String], required: true }
+    class TagsWithPagination < Pagination
+      expose :results, using: Entities::Tag, documentation: { type: Entities::Tag, is_array: true, required: true }
     end
   end
 end

data/lib/mihari/entities/whois.rb

@@ -11,6 +11,7 @@ module Mihari
       expose :contacts, documentation: { type: Hash, is_array: true, required: true } do |whois_record, _options|
         whois_record.contacts.map(&:to_camelback_keys)
       end
+      expose :created_at, documentation: { type: DateTime, required: true }, as: :createdAt
     end
   end
 end

data/lib/mihari/errors.rb

@@ -5,14 +5,12 @@ module Mihari
 
   class ValueError < Error; end
 
-  class TypeError < Error; end
-
   class RetryableError < Error; end
 
-  class FileNotFoundError < Error; end
-
   class ConfigurationError < Error; end
 
+  class IntegrityError < Error; end
+
   # errors for HTTP interactions
   class HTTPError < Error; end
 

data/lib/mihari/http.rb

@@ -31,6 +31,7 @@ module Mihari
     #
     class Factory
       class << self
+        USER_AGENT = "mihari/#{Mihari::VERSION}".freeze
         #
         # @param [Integer, nil] timeout
         # @param [Hash] headers
@@ -41,6 +42,9 @@ module Mihari
         # @param [Object] raise_exception
         def build(headers: {}, timeout: nil, raise_exception: true)
           client = raise_exception ? ::HTTP.use(:better_error) : ::HTTP
+
+          headers["User-Agent"] ||= USER_AGENT
+
           client = client.headers(headers)
           client = client.timeout(timeout) unless timeout.nil?
           client

data/lib/mihari/models/alert.rb

@@ -6,63 +6,31 @@ module Mihari
     # Alert model
     #
     class Alert < ActiveRecord::Base
-      has_many :taggings, dependent: :destroy
-      has_many :artifacts, dependent: :destroy
-      has_many :tags, through: :taggings
-
       belongs_to :rule
 
-      class << self
-        #
-        # Search alerts
-        #
-        # @param [Mihari::Structs::Filters::Alert::SearchFilterWithPagination] filter
-        #
-        # @return [Array<Alert>]
-        #
-        def search(filter)
-          limit = filter.limit.to_i
-          raise ArgumentError, "limit should be bigger than zero" unless limit.positive?
-
-          page = filter.page.to_i
-          raise ArgumentError, "page should be bigger than zero" unless page.positive?
-
-          offset = (page - 1) * limit
-
-          relation = build_relation(filter.without_pagination)
-          relation.limit(limit).offset(offset).order(id: :desc)
-        end
-
-        #
-        # Count alerts
-        #
-        # @param [Mihari::Structs::Filters::Alert::SearchFilter] filter
-        #
-        # @return [Integer]
-        #
-        def count(filter)
-          relation = build_relation(filter)
-          relation.distinct("alerts.id").count
-        end
-
-        private
-
-        #
-        # @param [Mihari::Structs::Filters::Alert::SearchFilter] filter
-        #
-        # @return [Mihari::Models::Alert]
-        #
-        def build_relation(filter)
-          relation = eager_load(:artifacts, :tags)
+      has_many :artifacts, dependent: :destroy
+      has_many :tags, through: :rule
+
+      include SearchCop
+      include Concerns::Searchable
+
+      search_scope :search do
+        attributes :id, :created_at, "rule.id", "rule.title", "rule.description"
+        attributes "artifact.data" => "artifacts.data"
+        attributes "artifact.data_type" => "artifacts.data_type"
+        attributes "artifact.source" => "artifacts.source"
+        attributes "artifact.query" => "artifacts.query"
+        attributes tag: "tags.name"
+      end
 
-          relation = relation.where(artifacts: { data: filter.artifact }) if filter.artifact
-          relation = relation.where(tags: { name: filter.tag }) if filter.tag
-          relation = relation.where(rule_id: filter.rule_id) if filter.rule_id
-          relation = relation.where("alerts.created_at >= ?", filter.from_at) if filter.from_at
-          relation = relation.where("alerts.created_at <= ?", filter.to_at) if filter.to_at
+      class << self
+        # @!method search_by_filter(filter)
+        #   @param [Mihari::Structs::Filters::Search] filter
+        #   @return [Array<Mihari::Models::Alert>]
 
-          relation
-        end
+        # @!method count_by_filter(filter)
+        #   @param [Mihari::Structs::Filters::Search] filter
+        #   @return [Integer]
       end
     end
   end