mihari 4.6.1 → 4.7.2

data/lib/mihari/models/whois.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "whois-parser"
-
 module Mihari
   class WhoisRecord < ActiveRecord::Base
     belongs_to :artifact
@@ -17,100 +15,7 @@ module Mihari
       # @return [WhoisRecord, nil]
       #
       def build_by_domain(domain)
-        domain = PublicSuffix.domain(domain)
-
-        # check memo
-        if @memo.key?(domain)
-          whois_record = @memo[domain]
-          # return clone of the record
-          return whois_record.dup
-        end
-
-        record = Whois.whois(domain)
-        parser = record.parser
-
-        return nil if parser.available?
-
-        whois_record = new(
-          domain: domain,
-          created_on: get_created_on(parser),
-          updated_on: get_updated_on(parser),
-          expires_on: get_expires_on(parser),
-          registrar: get_registrar(parser),
-          contacts: get_contacts(parser)
-        )
-        # set memo
-        @memo[domain] = whois_record
-        whois_record
-      rescue Whois::Error, Whois::ParserError, Timeout::Error
-        nil
-      end
-
-      private
-
-      #
-      # Get created_on
-      #
-      # @param [::Whois::Parser:] parser
-      #
-      # @return [Date, nil]
-      #
-      def get_created_on(parser)
-        parser.created_on
-      rescue ::Whois::AttributeNotImplemented
-        nil
-      end
-
-      #
-      # Get updated_on
-      #
-      # @param [::Whois::Parser:] parser
-      #
-      # @return [Date, nil]
-      #
-      def get_updated_on(parser)
-        parser.updated_on
-      rescue ::Whois::AttributeNotImplemented
-        nil
-      end
-
-      #
-      # Get expires_on
-      #
-      # @param [::Whois::Parser:] parser
-      #
-      # @return [Date, nil]
-      #
-      def get_expires_on(parser)
-        parser.expires_on
-      rescue ::Whois::AttributeNotImplemented
-        nil
-      end
-
-      #
-      # Get registrar
-      #
-      # @param [::Whois::Parser:] parser
-      #
-      # @return [Hash, nil]
-      #
-      def get_registrar(parser)
-        parser.registrar&.to_h
-      rescue ::Whois::AttributeNotImplemented
-        nil
-      end
-
-      #
-      # Get contacts
-      #
-      # @param [::Whois::Parser:] parser
-      #
-      # @return [Array[Hash], nil]
-      #
-      def get_contacts(parser)
-        parser.contacts.map(&:to_h)
-      rescue ::Whois::AttributeNotImplemented
-        nil
+        Enrichers::Whois.query domain
       end
     end
   end

data/lib/mihari/schemas/enricher.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Schemas
+    Enricher = Dry::Schema.Params do
+      required(:enricher).value(Types::EnricherTypes)
+    end
+  end
+end

data/lib/mihari/schemas/rule.rb

@@ -2,6 +2,7 @@
 
 require "mihari/schemas/analyzer"
 require "mihari/schemas/emitter"
+require "mihari/schemas/enricher"
 
 module Mihari
   module Schemas
@@ -20,6 +21,8 @@ module Mihari
 
       optional(:emitters).value(:array).each { Emitter | MISP | TheHive | Slack | HTTP }
 
+      optional(:enrichers).value(:array).each(Enricher)
+
       optional(:allowed_data_types).value(array[Types::DataTypes]).default(ALLOWED_DATA_TYPES)
       optional(:disallowed_data_values).value(array[:string]).default([])
 
@@ -35,6 +38,9 @@ module Mihari
         emitters = h[:emitters]
         h[:emitters] = emitters || DEFAULT_EMITTERS
 
+        enrichers = h[:enrichers]
+        h[:enrichers] = enrichers || DEFAULT_ENRICHERS
+
         h
       end
     end

data/lib/mihari/structs/filters.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Structs
+    module Filters
+      module Alert
+        class SearchFilter < Dry::Struct
+          attribute? :artifact_data, Types::String.optional
+          attribute? :description, Types::String.optional
+          attribute? :source, Types::String.optional
+          attribute? :tag_name, Types::String.optional
+          attribute? :title, Types::String.optional
+          attribute? :from_at, Types::DateTime.optional
+          attribute? :to_at, Types::DateTime.optional
+          attribute? :asn, Types::Int.optional
+          attribute? :dns_record, Types::String.optional
+          attribute? :reverse_dns_name, Types::String.optional
+
+          def valid_artifact_filters?
+            !(artifact_data || asn || dns_record || reverse_dns_name).nil?
+          end
+        end
+
+        class SearchFilterWithPagination < SearchFilter
+          attribute? :page, Types::Int.default(1)
+          attribute? :limit, Types::Int.default(10)
+
+          def without_pagination
+            SearchFilter.new(
+              artifact_data: artifact_data,
+              description: description,
+              from_at: from_at,
+              source: source,
+              tag_name: tag_name,
+              title: title,
+              to_at: to_at,
+              asn: asn,
+              dns_record: dns_record,
+              reverse_dns_name: reverse_dns_name
+            )
+          end
+        end
+      end
+
+      module Rule
+        class SearchFilter < Dry::Struct
+          attribute? :description, Types::String.optional
+          attribute? :tag_name, Types::String.optional
+          attribute? :title, Types::String.optional
+          attribute? :from_at, Types::DateTime.optional
+          attribute? :to_at, Types::DateTime.optional
+        end
+
+        class SearchFilterWithPagination < SearchFilter
+          attribute? :page, Types::Int.default(1)
+          attribute? :limit, Types::Int.default(10)
+
+          def without_pagination
+            SearchFilter.new(
+              description: description,
+              from_at: from_at,
+              tag_name: tag_name,
+              title: title,
+              to_at: to_at
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mihari/structs/google_public_dns.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Structs
+    module GooglePublicDNS
+      INT_TYPE_TO_TYPE = {
+        1 => "A",
+        2 => "NS",
+        5 => "CNAME",
+        16 => "TXT",
+        28 => "AAAA"
+      }
+
+      class Answer < Dry::Struct
+        attribute :name, Types::String
+        attribute :data, Types::String
+        attribute :resource_type, Types::String
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          resource_type = INT_TYPE_TO_TYPE[d.fetch("type")]
+          new(
+            name: d.fetch("name"),
+            data: d.fetch("data"),
+            resource_type: resource_type
+          )
+        end
+      end
+
+      class Response < Dry::Struct
+        attribute :answers, Types.Array(Answer)
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            answers: d.fetch("Answer", []).map { |x| Answer.from_dynamic!(x) }
+          )
+        end
+      end
+    end
+  end
+end

data/lib/mihari/structs/ipinfo.rb

@@ -6,8 +6,8 @@ module Mihari
       class Response < Dry::Struct
         attribute :ip, Types::String
         attribute :hostname, Types::String.optional
-        attribute :loc, Types::String
-        attribute :country_code, Types::String
+        attribute :loc, Types::String.optional
+        attribute :country_code, Types::String.optional
         attribute :asn, Types::Integer.optional
 
         class << self
@@ -23,9 +23,9 @@ module Mihari
 
             new(
               ip: d.fetch("ip"),
-              loc: d.fetch("loc"),
+              loc: d["loc"],
               hostname: d["hostname"],
-              country_code: d.fetch("country"),
+              country_code: d["country"],
               asn: asn
             )
           end

data/lib/mihari/structs/rule.rb

@@ -1,186 +1,236 @@
 # frozen_string_literal: true
 
+require "date"
+require "erb"
+require "pathname"
+require "yaml"
+
 module Mihari
   module Structs
-    module Rule
-      class SearchFilter < Dry::Struct
-        attribute? :description, Types::String.optional
-        attribute? :tag_name, Types::String.optional
-        attribute? :title, Types::String.optional
-        attribute? :from_at, Types::DateTime.optional
-        attribute? :to_at, Types::DateTime.optional
+    class Rule
+      # @return [Hash]
+      attr_reader :data
+
+      # @return [String]
+      attr_reader :yaml
+
+      # @return [Array, nil]
+      attr_reader :errors
+
+      # @return [String]
+      attr_writer :id
+
+      def initialize(data, yaml)
+        @data = data.deep_symbolize_keys
+        @yaml = yaml
+
+        @errors = nil
+        @no_method_error = nil
+
+        validate
       end
 
-      class SearchFilterWithPagination < SearchFilter
-        attribute? :page, Types::Int.default(1)
-        attribute? :limit, Types::Int.default(10)
-
-        def without_pagination
-          SearchFilter.new(
-            description: description,
-            from_at: from_at,
-            tag_name: tag_name,
-            title: title,
-            to_at: to_at
-          )
+      #
+      # @return [Boolean]
+      #
+      def errors?
+        return false if @errors.nil?
+
+        !@errors.empty?
+      end
+
+      #
+      # @return [Array<String>]
+      #
+      def error_messages
+        return [] if @errors.nil?
+
+        @errors.messages.filter_map do |message|
+          path = message.path.map(&:to_s).join
+          "#{path} #{message.text}"
+        rescue NoMethodError
+          nil
         end
       end
 
-      class Rule
-        # @return [Hash]
-        attr_reader :data
+      def validate
+        begin
+          contract = Schemas::RuleContract.new
+          result = contract.call(data)
+        rescue NoMethodError => e
+          @no_method_error = e
+          return
+        end
+
+        @data = result.to_h
+        @errors = result.errors
+      end
 
-        # @return [String]
-        attr_reader :yaml
+      def validate!
+        raise RuleValidationError, "Data should be a hash" unless data.is_a?(Hash)
+        raise RuleValidationError, error_messages.join("\n") if errors?
+        raise RuleValidationError, "Something wrong with queries, emitters or enrichers." unless @no_method_error.nil?
+      rescue RuleValidationError => e
+        message = "Failed to parse the input as a rule"
+        message += ": #{e.message}" unless e.message.empty?
+        Mihari.logger.error message
 
-        # @return [Array, nil]
-        attr_reader :errors
+        raise e
+      end
 
-        # @return [String]
-        attr_writer :id
+      def [](key)
+        data[key.to_sym]
+      end
 
-        def initialize(data, yaml)
-          @data = data.deep_symbolize_keys
-          @yaml = yaml
+      #
+      # @return [String]
+      #
+      def id
+        @id ||= data[:id] || UUIDTools::UUID.md5_create(UUIDTools::UUID_URL_NAMESPACE, data.to_yaml).to_s
+      end
 
-          @errors = nil
-          @no_method_error = nil
+      #
+      # @return [String]
+      #
+      def title
+        @title ||= data[:title]
+      end
 
-          validate
-        end
+      #
+      # @return [String]
+      #
+      def description
+        @description ||= data[:description]
+      end
 
-        #
-        # @return [Boolean]
-        #
-        def errors?
-          return false if @errors.nil?
+      #
+      # @return [Mihari::Rule]
+      #
+      def to_model
+        rule = Mihari::Rule.find(id)
+
+        rule.title = title
+        rule.description = description
+        rule.data = data
+        rule.yaml = yaml
+
+        rule
+      rescue ActiveRecord::RecordNotFound
+        Mihari::Rule.new(
+          id: id,
+          title: title,
+          description: description,
+          data: data,
+          yaml: yaml
+        )
+      end
 
-          !@errors.empty?
-        end
+      #
+      # @return [Mihari::Analyzers::Rule]
+      #
+      def to_analyzer
+        analyzer = Mihari::Analyzers::Rule.new(
+          title: self[:title],
+          description: self[:description],
+          tags: self[:tags],
+          queries: self[:queries],
+          allowed_data_types: self[:allowed_data_types],
+          disallowed_data_values: self[:disallowed_data_values],
+          emitters: self[:emitters],
+          enrichers: self[:enrichers],
+          id: id
+        )
+        analyzer.ignore_old_artifacts = self[:ignore_old_artifacts]
+        analyzer.ignore_threshold = self[:ignore_threshold]
+
+        analyzer
+      end
+
+      class << self
+        include Mixins::Database
 
         #
-        # @return [Array<String>]
+        # @param [Mihari::Rule] model
         #
-        def error_messages
-          return [] if @errors.nil?
+        # @return [Mihari::Structs::Rule]
+        #
+        def from_model(model)
+          data = model.data.deep_symbolize_keys
+          # set ID if YAML data do not have ID
+          data[:id] = model.id unless data.key?(:id)
 
-          @errors.messages.map do |message|
-            path = message.path.map(&:to_s).join
-            "#{path} #{message.text}"
-          end
+          Structs::Rule.new(data, model.yaml)
         end
 
-        def validate
-          begin
-            contract = Schemas::RuleContract.new
-            result = contract.call(data)
-          rescue NoMethodError => e
-            @no_method_error = e
-            return
-          end
+        #
+        # @param [String] yaml
+        #
+        # @return [Mihari::Structs::Rule]
+        # @param [String, nil] id
+        #
+        def from_yaml(yaml, id: nil)
+          data = load_erb_yaml(yaml)
+          # set ID if id is given & YAML data do not have ID
+          data[:id] = id if !id.nil? && !data.key?(:id)
 
-          @data = result.to_h
-          @errors = result.errors
+          Structs::Rule.new(data, yaml)
         end
 
-        def validate!
-          raise RuleValidationError, "Data should be a hash" unless data.is_a?(Hash)
+        #
+        # @param [String] path_or_id Path to YAML file or YAML string or ID of a rule in the database
+        #
+        # @return [Mihari::Structs::Rule]
+        #
+        def from_path_or_id(path_or_id)
+          yaml = nil
 
-          raise RuleValidationError, error_messages.join("\n") if errors?
+          yaml = load_yaml_from_file(path_or_id) if File.exist?(path_or_id)
+          yaml = load_yaml_from_db(path_or_id) if yaml.nil?
 
-          raise RuleValidationError, "Something wrong with queries or emitters." unless @no_method_error.nil?
+          Structs::Rule.from_yaml yaml
         end
 
-        def [](key)
-          data[key.to_sym]
-        end
+        private
 
         #
-        # @return [String]
+        # Load ERR templated YAML
         #
-        def id
-          @id ||= data[:id] || UUIDTools::UUID.md5_create(UUIDTools::UUID_URL_NAMESPACE, data.to_yaml).to_s
-        end
-
+        # @param [String] yaml
         #
-        # @return [String]
+        # @return [Hash]
         #
-        def title
-          @title ||= data[:title]
+        def load_erb_yaml(yaml)
+          YAML.safe_load(ERB.new(yaml).result, permitted_classes: [Date, Symbol], symbolize_names: true)
+        rescue Psych::SyntaxError => e
+          raise YAMLSyntaxError, e.message
         end
 
         #
-        # @return [String]
+        # Load YAML string from path
         #
-        def description
-          @description ||= data[:description]
-        end
-
+        # @param [String] path
         #
-        # @return [Mihari::Rule]
+        # @return [String, nil]
         #
-        def to_model
-          rule = Mihari::Rule.find(id)
+        def load_yaml_from_file(path)
+          return nil unless Pathname(path).exist?
 
-          rule.title = title
-          rule.description = description
-          rule.data = data
-          rule.yaml = yaml
-
-          rule
-        rescue ActiveRecord::RecordNotFound
-          Mihari::Rule.new(
-            id: id,
-            title: title,
-            description: description,
-            data: data,
-            yaml: yaml
-          )
+          File.read path
         end
 
         #
-        # @return [Mihari::Analyzers::Rule]
+        # Load YAML string from the database
         #
-        def to_analyzer
-          analyzer = Mihari::Analyzers::Rule.new(
-            title: self[:title],
-            description: self[:description],
-            tags: self[:tags],
-            queries: self[:queries],
-            allowed_data_types: self[:allowed_data_types],
-            disallowed_data_values: self[:disallowed_data_values],
-            emitters: self[:emitters],
-            id: id
-          )
-          analyzer.ignore_old_artifacts = self[:ignore_old_artifacts]
-          analyzer.ignore_threshold = self[:ignore_threshold]
-
-          analyzer
-        end
-
-        class << self
-          include Mixins::Rule
-
-          #
-          # @param [Mihari::Rule] model
-          #
-          # @return [Mihari::Structs::Rule::Rule]
-          #
-          def from_model(model)
-            data = model.data.deep_symbolize_keys
-            data[:id] = model.id unless data.key?(:id)
-
-            Structs::Rule::Rule.new(data, model.yaml)
-          end
-
-          #
-          # @param [String] yaml
-          #
-          # @return [Mihari::Structs::Rule::Rule]
-          #
-          def from_yaml(yaml)
-            data = load_erb_yaml(yaml)
-            Structs::Rule::Rule.new(data, yaml)
+        # @param [String] id <description>
+        #
+        # @return [Hash]
+        #
+        def load_yaml_from_db(id)
+          with_db_connection do
+            rule = Mihari::Rule.find(id)
+            rule.yaml || rule.symbolized_data.to_yaml
+          rescue ActiveRecord::RecordNotFound
+            raise ArgumentError, "ID:#{id} is not found in the database"
           end
         end
       end

data/lib/mihari/types.rb

@@ -21,5 +21,12 @@ module Mihari
       "database",
       "webhook"
     )
+
+    EnricherTypes = Types::String.enum(
+      "whois",
+      "ipinfo",
+      "shodan",
+      "google_public_dns"
+    )
   end
 end

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "4.6.1"
+  VERSION = "4.7.2"
 end

data/lib/mihari/web/endpoints/alerts.rb

@@ -43,7 +43,7 @@ module Mihari
           # symbolize hash keys
           filter = filter.to_h.symbolize_keys
 
-          search_filter_with_pagenation = Structs::Alert::SearchFilterWithPagination.new(**filter)
+          search_filter_with_pagenation = Structs::Filters::Alert::SearchFilterWithPagination.new(**filter)
           alerts = Mihari::Alert.search(search_filter_with_pagenation)
           total = Mihari::Alert.count(search_filter_with_pagenation.without_pagination)