mihari 7.6.2 → 7.6.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c5fff977863ca54dae5b645e3587d8f09f0583df44fca258ac4f1c8f3455a74e
-  data.tar.gz: 3c45cf5405737aaddbd79b42590509f155b3be13c3a0645057cc446666728655
+  metadata.gz: 12856ce5fee6e623a5579cab99a03629c611db1f65e087b04167cae69f870647
+  data.tar.gz: 28577ad798c7033ead6aaeced545a9401f17d0a31d29551950c3be583bc107ea
 SHA512:
-  metadata.gz: d189d79161738b06e2ee970b835b953bd968c4c4500b7cf589fcf3c2ac31a49f46a9d2fcf3581818484f9daf0b72a87a1d3cd39084d39e4a9bcfdf4ab0edc1a4
-  data.tar.gz: 463d12e871297b9d5be403139720fc51d09bd51e0c41855d47cbe889de086f2a5a2d2fe25fa592f81ddec1b6e56195b9878372c1fc8955f23357b72d09bbdce0
+  metadata.gz: 68d780eca473d2b081c3c9bb7a60c9c5bfa371a0c6e08a307ceaa3138f5097b70606902e657aab22d1013a5033904bd7135595812d76a0554bde38f3b75917b6
+  data.tar.gz: 6b0ab8b458109b4c496298560772bd44e845a96c97ab89ec3191416855a66c4de92277d66d63607b17a857d35be10343adbdd71183616f64e7b984b617cde32a

data/Rakefile

@@ -40,9 +40,7 @@ def build_swagger_doc(path)
     recursive_delete json, key
   end
 
-  f = File.open(path, "w")
-  f.write json.to_yaml
-  f.close
+  File.write(path, json.to_yaml)
 end
 
 namespace :build do

data/lib/mihari/commands/rule.rb

@@ -26,14 +26,37 @@ module Mihari
               end
             end
 
-            desc "validate PATH", "Validate a rule"
+            desc "validate PATH", "Validate rule(s)"
             #
-            # Validate format of a rule
+            # Validate rule(s)
+            #
+            # @param [Array<String>] paths
+            #
+            def validate(*paths)
+              # @type [Array<Mihari::ValidationError>]
+              errors = paths.flat_map { |path| Dir.glob(path) }.map do |path|
+                Dry::Monads::Try[ValidationError] { Mihari::Rule.from_file(path) }
+              end.filter_map do |result|
+                result.exception if result.error?
+              end
+              return if errors.empty?
+
+              errors.each do |error|
+                data = Entities::ErrorMessage.represent(message: error.message, detail: error.detail)
+                warn JSON.pretty_generate(data.as_json)
+              end
+
+              exit 1
+            end
+
+            desc "format PATH", "format a rule"
+            #
+            # Format a rule file
             #
             # @param [String] path
             #
-            def validate(path)
-              rule = Dry::Monads::Try[ValidationError] { Mihari::Rule.from_yaml File.read(path) }.value!
+            def format(path)
+              rule = Dry::Monads::Try[ValidationError] { Mihari::Rule.from_file(path) }.value!
               puts rule.data.to_yaml
             end
 

data/lib/mihari/config.rb

@@ -40,6 +40,7 @@ module Mihari
       zoomeye_api_key: nil,
       # sidekiq
       sidekiq_redis_url: nil,
+      sidekiq_retry: 0,
       # others
       hide_config_values: true,
       ignore_error: false,
@@ -174,6 +175,9 @@ module Mihari
     # @!attribute [r] sidekiq_redis_url
     #   @return [URI, nil]
 
+    # @!attribute [r] sidekiq_retry
+    #   @return [Integer]
+
     def database_url=(val)
       super(URI(val.to_s))
     end

data/lib/mihari/database.rb

@@ -6,7 +6,7 @@ ActiveSupport::Inflector.inflections(:en) { |inflect| inflect.acronym "CPE" }
 #
 # Mihari v7 DB schema
 #
-class V7Schema < ActiveRecord::Migration[7.1]
+class V7Schema < ActiveRecord::Migration[7.2]
   def change
     create_table :rules, id: :string, if_not_exists: true do |t|
       t.string :title, null: false

data/lib/mihari/enrichers/whois.rb

@@ -16,7 +16,9 @@ module Mihari
       def call(artifact)
         return if artifact.domain.nil?
 
-        artifact.whois_record ||= memoized_lookup(PublicSuffix.domain(artifact.domain))
+        artifact.tap do |tapped|
+          tapped.whois_record ||= memoized_lookup(PublicSuffix.domain(artifact.domain))
+        end
       end
 
       private

data/lib/mihari/models/artifact.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "ostruct"
+
 module Mihari
   module Models
     #
@@ -158,10 +160,7 @@ module Mihari
       # @return [Boolean] true if it is unique. Otherwise false.
       #
       def unique?(base_time: nil, artifact_ttl: nil)
-        artifact = self.class.joins(:alert).where(
-          data:,
-          alert: {rule_id:}
-        ).order(created_at: :desc).first
+        artifact = self.class.joins(:alert).where(data:, alert: {rule_id:}).order(created_at: :desc).first
         return true if artifact.nil?
 
         # check whether the artifact is decayed or not
@@ -179,7 +178,32 @@ module Mihari
       end
 
       def enrich
-        callable_enrichers.each { |enricher| enricher.result self }
+        enrich_by_enrichers callable_enrichers
+      end
+
+      #
+      # @param [Array<Mihari::Enrichers::Base>] enrichers
+      # @param [Boolean] parallel
+      #
+      # @return [Mihari::Models::Artifact]
+      #
+      def enrich_by_enrichers(enrichers)
+        # NOTE: doing parallel with ActiveRecord objects is troublesome (e.g. connection issue, etc.)
+        #       so converting the object to an OpenStruct object
+        s = struct
+        results = Parallel.map(enrichers) { |enricher| enricher.result s }
+        enriched = results.compact.map { |result| result.value_or(nil) }.compact
+
+        self.dns_records = enriched.map(&:dns_records).flatten.compact
+        self.cpes = enriched.map(&:cpes).flatten.compact
+        self.ports = enriched.map(&:ports).flatten.compact
+        self.vulnerabilities = enriched.map(&:vulnerabilities).flatten.compact
+
+        self.autonomous_system = enriched.map(&:autonomous_system).compact.first
+        self.geolocation = enriched.map(&:geolocation).compact.first
+        self.whois_record = enriched.map(&:whois_record).compact.first
+
+        self
       end
 
       #
@@ -195,6 +219,18 @@ module Mihari
         end
       end
 
+      def struct
+        OpenStruct.new(attributes).tap do |s|
+          s.domain = domain
+          s.cpes ||= []
+          s.dns_records ||= []
+          s.ports ||= []
+          s.reverse_dns_names ||= []
+          s.vulnerabilities ||= []
+          s.tags ||= []
+        end
+      end
+
       class << self
         # @!method search_by_filter(filter)
         #   @param [Mihari::Structs::Filters::Search] filter
@@ -212,7 +248,7 @@ module Mihari
       #
       def callable_enrichers
         @callable_enrichers ||= Mihari.enrichers.map(&:new).select do |enricher|
-          enricher.callable?(self)
+          enricher.callable? self
         end
       end
 

data/lib/mihari/rule.rb

@@ -5,6 +5,9 @@ module Mihari
     include Concerns::FalsePositiveNormalizable
     include Concerns::FalsePositiveValidatable
 
+    # @return [String, nil]
+    attr_reader :path_or_id
+
     # @return [Hash]
     attr_reader :data
 
@@ -19,9 +22,11 @@ module Mihari
     #
     # @param [Hash] data
     #
-    def initialize(**data)
+    # @param [Object] path_or_id
+    def initialize(path_or_id: nil, **data)
       super()
 
+      @path_or_id = path_or_id
       @data = data.deep_symbolize_keys
       @errors = nil
       @base_time = Time.now.utc
@@ -173,10 +178,7 @@ module Mihari
     #
     def enriched_artifacts
       @enriched_artifacts ||= Parallel.map(unique_artifacts) do |artifact|
-        artifact.tap do |tapped|
-          # NOTE: To apply changes correctly, enrichers should be applied to an artifact serially
-          enrichers.each { |enricher| enricher.result(tapped) }
-        end
+        artifact.enrich_by_enrichers enrichers
       end
     end
 
@@ -251,16 +253,29 @@ module Mihari
     end
 
     class << self
+      #
+      # Load rule from YAML file
+      #
+      # @param [String] path
+      #
+      # @return [Mihari::Rule]
+      #
+      def from_file(path)
+        yaml = File.read(path)
+        from_yaml(yaml, path: path)
+      end
+
       #
       # Load rule from YAML string
       #
       # @param [String] yaml
+      # @param [String, nil] path
       #
       # @return [Mihari::Rule]
       #
-      def from_yaml(yaml)
+      def from_yaml(yaml, path: nil)
         data = YAML.safe_load(ERB.new(yaml).result, permitted_classes: [Date, Symbol])
-        new(**data)
+        new(path_or_id: path, **data)
       end
 
       #
@@ -269,7 +284,7 @@ module Mihari
       # @return [Mihari::Rule]
       #
       def from_model(model)
-        new(**model.data)
+        new(path_or_id: model.id, **model.data)
       end
     end
 
@@ -409,7 +424,7 @@ module Mihari
       @data = result.to_h
       @errors = result.errors
 
-      raise ValidationError.new("Validation failed", errors) if errors?
+      raise ValidationError.new("#{path_or_id}: validation failed", errors) if errors?
     end
   end
 end

data/lib/mihari/sidekiq/application.rb

@@ -6,6 +6,7 @@ require "mihari/sidekiq/jobs"
 
 Sidekiq.configure_server do |config|
   config.redis = {url: Mihari.config.sidekiq_redis_url.to_s}
+  config.default_job_options = {retry: Mihari.config.sidekiq_retry}
 end
 
 Sidekiq.configure_client do |config|

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "7.6.2"
+  VERSION = "7.6.4"
 end