mihari 4.7.1 → 4.7.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 38f68ec091b2469095f321f104c1e238599c56afa7f55763920cfeef0df03ca0
-  data.tar.gz: 625bbd99f92e4af768b76db15ab72cd53dba6a04150acc3ddfe7feb7d1a82bf7
+  metadata.gz: 7eeecbacefc194f5d2e9356fe8f17884c358b45bb3b7ccbd83a47aa3567ae1f6
+  data.tar.gz: d99abc5ef4368bf1a23a48cf0f15a64a33d3539313a624d489887e86a8cc564c
 SHA512:
-  metadata.gz: fe847e37658ede26d1910e8c73ee1726e2b32335571e03c97c891b2d799380397ed1d32de7e4ed31ae57c861b4a6aa74433454d4871e38e2137100ce342894a1
-  data.tar.gz: 688c3adcf6ebb754d02b0fecde033602b47fe23ae11fffaff329ce53b93c268ab274319fdb630c819163856ce9708f2658bfe6fe7790e54ab1488e8d2af90245
+  metadata.gz: e0347f1c03a949d7e3c85cb9a615247c41bea2a0ac8288f0e3c7a8f9bd5d93ad8f952c99f3b595bf19997fe10e32490898b58a858f63f402a6082344f68cdb87
+  data.tar.gz: a507cfb86593c602ba46ef59f279a17ed1a34e433c8e34898a872048306733d26fb1a6546f1ae96605d524ccd80c3c3a0487ab4b4ad66219c0e3a63215ce8c6e

data/lib/mihari/analyzers/rule.rb

@@ -39,7 +39,6 @@ module Mihari
 
     class Rule < Base
       include Mixins::DisallowedDataValue
-      include Mixins::Rule
 
       option :title
       option :description

data/lib/mihari/commands/init.rb

@@ -3,8 +3,6 @@
 module Mihari
   module Commands
     module Initialization
-      include Mixins::Rule
-
       def self.included(thor)
         thor.class_eval do
           desc "rule", "Create a rule file"
@@ -21,6 +19,31 @@ module Mihari
 
             Mihari.logger.info "The rule file is initialized as #{filename}."
           end
+
+          no_commands do
+            #
+            # Returns a template for rule
+            #
+            # @return [String] A template for rule
+            #
+            def rule_template
+              rule = Structs::Rule.from_path_or_id File.expand_path("../templates/rule.yml.erb", __dir__)
+              rule.yaml
+            end
+
+            #
+            # Create (blank) rule file
+            #
+            # @param [String] filename
+            # @param [Dry::Files] files
+            # @param [String] template
+            #
+            # @return [nil]
+            #
+            def initialize_rule_yaml(filename, files = Dry::Files.new, template: rule_template)
+              files.write(filename, template)
+            end
+          end
         end
       end
     end

data/lib/mihari/commands/search.rb

@@ -4,7 +4,6 @@ module Mihari
   module Commands
     module Search
       include Mixins::Database
-      include Mixins::Rule
       include Mixins::ErrorNotification
 
       def self.included(thor)
@@ -12,14 +11,10 @@ module Mihari
           desc "search [RULE]", "Search by a rule"
           method_option :yes, type: :boolean, aliases: "-y", desc: "yes to overwrite the rule in the database"
           def search_by_rule(path_or_id)
-            rule = load_rule(path_or_id)
+            rule = Structs::Rule.from_path_or_id path_or_id
 
             # validate
-            begin
-              validate_rule! rule
-            rescue RuleValidationError => e
-              raise e
-            end
+            rule.validate!
 
             # check update
             id = rule.id

data/lib/mihari/commands/validator.rb

@@ -3,16 +3,21 @@
 module Mihari
   module Commands
     module Validator
-      include Mixins::Rule
-
       def self.included(thor)
         thor.class_eval do
-          desc "rule [PATH]", "Validate format of a rule file"
+          desc "rule [PATH]", "Validate rule file format"
+          #
+          # Validate format of a rule
+          #
+          # @param [String] path
+          #
+          # @return [nil]
+          #
           def rule(path)
-            rule = load_rule(path)
+            rule = Structs::Rule.from_path_or_id(path)
 
             begin
-              validate_rule! rule
+              rule.validate!
               Mihari.logger.info "Valid format. The input is parsed as the following:\n#{rule.data.to_yaml}"
             rescue RuleValidationError
               nil

data/lib/mihari/errors.rb

@@ -15,6 +15,8 @@ module Mihari
 
   class RuleValidationError < Error; end
 
+  class YAMLSyntaxError < Error; end
+
   class ConfigurationError < Error; end
 
   class HTTPError < Error; end

data/lib/mihari/models/alert.rb

@@ -12,7 +12,7 @@ module Mihari
       #
       # Search alerts
       #
-      # @param [Structs::Alert::SearchFilterWithPagination] filter
+      # @param [Structs::Filters::Alert::SearchFilterWithPagination] filter
       #
       # @return [Array<Alert>]
       #
@@ -58,6 +58,11 @@ module Mihari
 
       private
 
+      #
+      # @param [Structs::Filters::Alert::SearchFilter] filter
+      #
+      # @return [Mihari::Alert]
+      #
       def build_relation(filter)
         artifact_ids = []
         artifact = Artifact.includes(:autonomous_system, :dns_records, :reverse_dns_names)

data/lib/mihari/models/geolocation.rb

@@ -17,11 +17,9 @@ module Mihari
       def build_by_ip(ip)
         res = Enrichers::IPInfo.query(ip)
 
-        unless res.nil?
-          return new(country: NormalizeCountry(res.country_code, to: :short), country_code: res.country_code)
-        end
+        return nil if res&.country_code.nil?
 
-        nil
+        new(country: NormalizeCountry(res.country_code, to: :short), country_code: res.country_code)
       end
     end
   end

data/lib/mihari/models/port.rb

@@ -14,7 +14,7 @@ module Mihari
       #
       def build_by_ip(ip)
         res = Enrichers::Shodan.query(ip)
-        return if res.nil?
+        return [] if res.nil?
 
         res.ports.map { |port| new(port: port) }
       end

data/lib/mihari/models/rule.rb

@@ -28,7 +28,7 @@ module Mihari
       #
       # Search rules
       #
-      # @param [Structs::Rule::SearchFilterWithPagination] filter
+      # @param [Structs::Filters::Rule::SearchFilterWithPagination] filter
       #
       # @return [Array<Rule>]
       #
@@ -51,7 +51,7 @@ module Mihari
       #
       # Count alerts
       #
-      # @param [Structs::Rule::SearchFilterWithPagination] filter
+      # @param [Structs::Filters::Rule::SearchFilterWithPagination] filter
       #
       # @return [Integer]
       #
@@ -62,6 +62,11 @@ module Mihari
 
       private
 
+      #
+      # @param [Structs::Filters::Rule::SearchFilter] filter
+      #
+      # @return [Mihari::Rule]
+      #
       def build_relation(filter)
         relation = self
         relation = relation.includes(alerts: :tags)

data/lib/mihari/structs/filters.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Structs
+    module Filters
+      module Alert
+        class SearchFilter < Dry::Struct
+          attribute? :artifact_data, Types::String.optional
+          attribute? :description, Types::String.optional
+          attribute? :source, Types::String.optional
+          attribute? :tag_name, Types::String.optional
+          attribute? :title, Types::String.optional
+          attribute? :from_at, Types::DateTime.optional
+          attribute? :to_at, Types::DateTime.optional
+          attribute? :asn, Types::Int.optional
+          attribute? :dns_record, Types::String.optional
+          attribute? :reverse_dns_name, Types::String.optional
+
+          def valid_artifact_filters?
+            !(artifact_data || asn || dns_record || reverse_dns_name).nil?
+          end
+        end
+
+        class SearchFilterWithPagination < SearchFilter
+          attribute? :page, Types::Int.default(1)
+          attribute? :limit, Types::Int.default(10)
+
+          def without_pagination
+            SearchFilter.new(
+              artifact_data: artifact_data,
+              description: description,
+              from_at: from_at,
+              source: source,
+              tag_name: tag_name,
+              title: title,
+              to_at: to_at,
+              asn: asn,
+              dns_record: dns_record,
+              reverse_dns_name: reverse_dns_name
+            )
+          end
+        end
+      end
+
+      module Rule
+        class SearchFilter < Dry::Struct
+          attribute? :description, Types::String.optional
+          attribute? :tag_name, Types::String.optional
+          attribute? :title, Types::String.optional
+          attribute? :from_at, Types::DateTime.optional
+          attribute? :to_at, Types::DateTime.optional
+        end
+
+        class SearchFilterWithPagination < SearchFilter
+          attribute? :page, Types::Int.default(1)
+          attribute? :limit, Types::Int.default(10)
+
+          def without_pagination
+            SearchFilter.new(
+              description: description,
+              from_at: from_at,
+              tag_name: tag_name,
+              title: title,
+              to_at: to_at
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mihari/structs/ipinfo.rb

@@ -6,8 +6,8 @@ module Mihari
       class Response < Dry::Struct
         attribute :ip, Types::String
         attribute :hostname, Types::String.optional
-        attribute :loc, Types::String
-        attribute :country_code, Types::String
+        attribute :loc, Types::String.optional
+        attribute :country_code, Types::String.optional
         attribute :asn, Types::Integer.optional
 
         class << self
@@ -23,9 +23,9 @@ module Mihari
 
             new(
               ip: d.fetch("ip"),
-              loc: d.fetch("loc"),
+              loc: d["loc"],
               hostname: d["hostname"],
-              country_code: d.fetch("country"),
+              country_code: d["country"],
               asn: asn
             )
           end

data/lib/mihari/structs/rule.rb

@@ -1,192 +1,236 @@
 # frozen_string_literal: true
 
+require "date"
+require "erb"
+require "pathname"
+require "yaml"
+
 module Mihari
   module Structs
-    module Rule
-      class SearchFilter < Dry::Struct
-        attribute? :description, Types::String.optional
-        attribute? :tag_name, Types::String.optional
-        attribute? :title, Types::String.optional
-        attribute? :from_at, Types::DateTime.optional
-        attribute? :to_at, Types::DateTime.optional
+    class Rule
+      # @return [Hash]
+      attr_reader :data
+
+      # @return [String]
+      attr_reader :yaml
+
+      # @return [Array, nil]
+      attr_reader :errors
+
+      # @return [String]
+      attr_writer :id
+
+      def initialize(data, yaml)
+        @data = data.deep_symbolize_keys
+        @yaml = yaml
+
+        @errors = nil
+        @no_method_error = nil
+
+        validate
       end
 
-      class SearchFilterWithPagination < SearchFilter
-        attribute? :page, Types::Int.default(1)
-        attribute? :limit, Types::Int.default(10)
-
-        def without_pagination
-          SearchFilter.new(
-            description: description,
-            from_at: from_at,
-            tag_name: tag_name,
-            title: title,
-            to_at: to_at
-          )
+      #
+      # @return [Boolean]
+      #
+      def errors?
+        return false if @errors.nil?
+
+        !@errors.empty?
+      end
+
+      #
+      # @return [Array<String>]
+      #
+      def error_messages
+        return [] if @errors.nil?
+
+        @errors.messages.filter_map do |message|
+          path = message.path.map(&:to_s).join
+          "#{path} #{message.text}"
+        rescue NoMethodError
+          nil
         end
       end
 
-      class Rule
-        # @return [Hash]
-        attr_reader :data
+      def validate
+        begin
+          contract = Schemas::RuleContract.new
+          result = contract.call(data)
+        rescue NoMethodError => e
+          @no_method_error = e
+          return
+        end
+
+        @data = result.to_h
+        @errors = result.errors
+      end
 
-        # @return [String]
-        attr_reader :yaml
+      def validate!
+        raise RuleValidationError, "Data should be a hash" unless data.is_a?(Hash)
+        raise RuleValidationError, error_messages.join("\n") if errors?
+        raise RuleValidationError, "Something wrong with queries, emitters or enrichers." unless @no_method_error.nil?
+      rescue RuleValidationError => e
+        message = "Failed to parse the input as a rule"
+        message += ": #{e.message}" unless e.message.empty?
+        Mihari.logger.error message
 
-        # @return [Array, nil]
-        attr_reader :errors
+        raise e
+      end
 
-        # @return [String]
-        attr_writer :id
+      def [](key)
+        data[key.to_sym]
+      end
 
-        def initialize(data, yaml)
-          @data = data.deep_symbolize_keys
-          @yaml = yaml
+      #
+      # @return [String]
+      #
+      def id
+        @id ||= data[:id] || UUIDTools::UUID.md5_create(UUIDTools::UUID_URL_NAMESPACE, data.to_yaml).to_s
+      end
 
-          @errors = nil
-          @no_method_error = nil
+      #
+      # @return [String]
+      #
+      def title
+        @title ||= data[:title]
+      end
 
-          validate
-        end
+      #
+      # @return [String]
+      #
+      def description
+        @description ||= data[:description]
+      end
 
-        #
-        # @return [Boolean]
-        #
-        def errors?
-          return false if @errors.nil?
+      #
+      # @return [Mihari::Rule]
+      #
+      def to_model
+        rule = Mihari::Rule.find(id)
+
+        rule.title = title
+        rule.description = description
+        rule.data = data
+        rule.yaml = yaml
+
+        rule
+      rescue ActiveRecord::RecordNotFound
+        Mihari::Rule.new(
+          id: id,
+          title: title,
+          description: description,
+          data: data,
+          yaml: yaml
+        )
+      end
 
-          !@errors.empty?
-        end
+      #
+      # @return [Mihari::Analyzers::Rule]
+      #
+      def to_analyzer
+        analyzer = Mihari::Analyzers::Rule.new(
+          title: self[:title],
+          description: self[:description],
+          tags: self[:tags],
+          queries: self[:queries],
+          allowed_data_types: self[:allowed_data_types],
+          disallowed_data_values: self[:disallowed_data_values],
+          emitters: self[:emitters],
+          enrichers: self[:enrichers],
+          id: id
+        )
+        analyzer.ignore_old_artifacts = self[:ignore_old_artifacts]
+        analyzer.ignore_threshold = self[:ignore_threshold]
+
+        analyzer
+      end
+
+      class << self
+        include Mixins::Database
 
         #
-        # @return [Array<String>]
+        # @param [Mihari::Rule] model
         #
-        def error_messages
-          return [] if @errors.nil?
+        # @return [Mihari::Structs::Rule]
+        #
+        def from_model(model)
+          data = model.data.deep_symbolize_keys
+          # set ID if YAML data do not have ID
+          data[:id] = model.id unless data.key?(:id)
 
-          @errors.messages.map do |message|
-            path = message.path.map(&:to_s).join
-            "#{path} #{message.text}"
-          end
+          Structs::Rule.new(data, model.yaml)
         end
 
-        def validate
-          begin
-            contract = Schemas::RuleContract.new
-            result = contract.call(data)
-          rescue NoMethodError => e
-            @no_method_error = e
-            return
-          end
+        #
+        # @param [String] yaml
+        #
+        # @return [Mihari::Structs::Rule]
+        # @param [String, nil] id
+        #
+        def from_yaml(yaml, id: nil)
+          data = load_erb_yaml(yaml)
+          # set ID if id is given & YAML data do not have ID
+          data[:id] = id if !id.nil? && !data.key?(:id)
 
-          @data = result.to_h
-          @errors = result.errors
+          Structs::Rule.new(data, yaml)
         end
 
-        def validate!
-          raise RuleValidationError, "Data should be a hash" unless data.is_a?(Hash)
+        #
+        # @param [String] path_or_id Path to YAML file or YAML string or ID of a rule in the database
+        #
+        # @return [Mihari::Structs::Rule]
+        #
+        def from_path_or_id(path_or_id)
+          yaml = nil
 
-          raise RuleValidationError, error_messages.join("\n") if errors?
+          yaml = load_yaml_from_file(path_or_id) if File.exist?(path_or_id)
+          yaml = load_yaml_from_db(path_or_id) if yaml.nil?
 
-          raise RuleValidationError, "Something wrong with queries, emitters or enrichers." unless @no_method_error.nil?
+          Structs::Rule.from_yaml yaml
         end
 
-        def [](key)
-          data[key.to_sym]
-        end
+        private
 
         #
-        # @return [String]
+        # Load ERR templated YAML
         #
-        def id
-          @id ||= data[:id] || UUIDTools::UUID.md5_create(UUIDTools::UUID_URL_NAMESPACE, data.to_yaml).to_s
-        end
-
+        # @param [String] yaml
         #
-        # @return [String]
+        # @return [Hash]
         #
-        def title
-          @title ||= data[:title]
+        def load_erb_yaml(yaml)
+          YAML.safe_load(ERB.new(yaml).result, permitted_classes: [Date, Symbol], symbolize_names: true)
+        rescue Psych::SyntaxError => e
+          raise YAMLSyntaxError, e.message
         end
 
         #
-        # @return [String]
+        # Load YAML string from path
         #
-        def description
-          @description ||= data[:description]
-        end
-
+        # @param [String] path
         #
-        # @return [Mihari::Rule]
+        # @return [String, nil]
         #
-        def to_model
-          rule = Mihari::Rule.find(id)
+        def load_yaml_from_file(path)
+          return nil unless Pathname(path).exist?
 
-          rule.title = title
-          rule.description = description
-          rule.data = data
-          rule.yaml = yaml
-
-          rule
-        rescue ActiveRecord::RecordNotFound
-          Mihari::Rule.new(
-            id: id,
-            title: title,
-            description: description,
-            data: data,
-            yaml: yaml
-          )
+          File.read path
         end
 
         #
-        # @return [Mihari::Analyzers::Rule]
-        #
-        def to_analyzer
-          analyzer = Mihari::Analyzers::Rule.new(
-            title: self[:title],
-            description: self[:description],
-            tags: self[:tags],
-            queries: self[:queries],
-            allowed_data_types: self[:allowed_data_types],
-            disallowed_data_values: self[:disallowed_data_values],
-            emitters: self[:emitters],
-            enrichers: self[:enrichers],
-            id: id
-          )
-          analyzer.ignore_old_artifacts = self[:ignore_old_artifacts]
-          analyzer.ignore_threshold = self[:ignore_threshold]
-
-          analyzer
-        end
-
-        class << self
-          include Mixins::Rule
-
-          #
-          # @param [Mihari::Rule] model
-          #
-          # @return [Mihari::Structs::Rule::Rule]
-          #
-          def from_model(model)
-            data = model.data.deep_symbolize_keys
-            # set ID if YAML data do not have ID
-            data[:id] = model.id unless data.key?(:id)
-
-            Structs::Rule::Rule.new(data, model.yaml)
-          end
-
-          #
-          # @param [String] yaml
-          #
-          # @return [Mihari::Structs::Rule::Rule]
-          # @param [String, nil] id
-          #
-          def from_yaml(yaml, id: nil)
-            data = load_erb_yaml(yaml)
-            # set ID if id is given & YAML data do not have ID
-            data[:id] = id if !id.nil? && !data.key?(:id)
-
-            Structs::Rule::Rule.new(data, yaml)
+        # Load YAML string from the database
+        #
+        # @param [String] id <description>
+        #
+        # @return [Hash]
+        #
+        def load_yaml_from_db(id)
+          with_db_connection do
+            rule = Mihari::Rule.find(id)
+            rule.yaml || rule.symbolized_data.to_yaml
+          rescue ActiveRecord::RecordNotFound
+            raise ArgumentError, "ID:#{id} is not found in the database"
           end
         end
       end

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "4.7.1"
+  VERSION = "4.7.2"
 end