mihari 7.6.1 → 7.6.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2fd0c07fe2661f57d6db5b409c527789c5e17d23e7691d5ac241ff5b36a20698
-  data.tar.gz: d7f2a832b3cb364aca8bdfad71cf762f72dc707892d2697ef8bb272bcb794b1d
+  metadata.gz: 0d9b88f5339ebe2064567127db7fac4a4e0c9ec56648c575de2c96fcb16a3af8
+  data.tar.gz: 15f840a496f423dff92dcc34e2f7c4ad39ed7b1a370c053a5f5cdf29c39f84a2
 SHA512:
-  metadata.gz: 39c5c92cf79721ed6d98b09a63a3d609eab8e8da9ba5badb1cd1a9f01fcb04d8f11edaa3b50fd7e0aed6dcb44f7befb77166e9bbf8399bac4d6592eafd943763
-  data.tar.gz: 7c06bd2ca9616605aa9601d198a413529f92205861dcafbfd86e84fa5bca6e2344d42ef558f27ee4c358f09778679fdaa2018954a6d34bfa534f2a3fa155042d
+  metadata.gz: f07c6c65db1dd47634e0f5919d1b6faa8250cc02d255226085c1fd88f6a32e78338a6ddeaa7b0f221e4593af81735635c4652cdfd46b415bfa0e5ecaf08cb0f1
+  data.tar.gz: 5d8411ddbee4e1e466e3521ec611ffba67e52ea3852e3cfce8f095978b60584b15cc541118064d2fd8bfa9e40c14720899efafa006f96543e7947fecbd5b76da

data/.rubocop.yml

@@ -1,5 +1,6 @@
-Style/StringLiterals:
-  EnforcedStyle: double_quotes
+AllCops:
+  TargetRubyVersion: 3.2
+  NewCops: enable
 Metrics/BlockLength:
   Max: 150
   Exclude:
@@ -12,13 +13,32 @@ Metrics/MethodLength:
 Metrics/AbcSize:
   Max: 50
 RSpec/MultipleMemoizedHelpers:
-  Max: 10
+  Max: 15
 RSpec/ExampleLength:
   Max: 20
-RSpec/FilePath:
-  SpecSuffixOnly: true
+RSpec/NestedGroups:
+  Max: 5
+RSpec/RepeatedExampleGroupDescription:
+  Enabled: false
+RSpec/ReceiveMessages:
+  Enabled: false
+RSpec/MultipleExpectations:
+  Enabled: false
+RSpec/SpecFilePathFormat:
+  Enabled: false
+FactoryBot/SyntaxMethods:
+  Enabled: false
 require:
+  - rubocop-capybara
   - rubocop-factory_bot
+  - rubocop-performance
   - rubocop-rake
   - rubocop-rspec
   - rubocop-yard
+  - standard
+  - standard-custom
+  - standard-performance
+inherit_gem:
+  standard: config/base.yml
+  standard-custom: config/base.yml
+  standard-performance: config/base.yml

data/README.md

@@ -27,6 +27,7 @@ Mihari supports the following services by default.
 - [SecurityTrails](https://securitytrails.com/)
 - [Shodan](https://shodan.io)
 - [urlscan.io](https://urlscan.io)
+- [Validin](https://validin.com)
 - [VirusTotal](http://virustotal.com) & [VirusTotal Intelligence](https://www.virustotal.com/gui/intelligence-overview)
 - [ZoomEye](https://zoomeye.org)
 

data/Rakefile

@@ -3,8 +3,9 @@
 require "time"
 
 require "rspec/core/rake_task"
-require "standard/rake"
+require "rubocop/rake_task"
 
+RuboCop::RakeTask.new
 RSpec::Core::RakeTask.new(:spec)
 
 task default: :spec
@@ -67,6 +68,7 @@ namespace :build do
   end
 end
 
+desc "Build including Swagger doc and frontend assets"
 task :build do
   Rake::Task["build:swagger"].invoke
   Rake::Task["build:frontend"].invoke

data/lefthook.yml

@@ -1,8 +1,8 @@
 pre-commit:
   commands:
-    standard:
+    rubocop:
       glob: "*.rb"
-      run: bundle exec standardrb --fix {staged_files}
+      run: bundle exec rubocop --fix {staged_files}
       stage_fixed: true
     eslint:
       root: "frontend/"
@@ -19,5 +19,5 @@ pre-commit:
       glob: "*.{js,ts,vue}"
       run: npm run type-check
     actionlint:
-      glob: ".github/workflows/*.yaml"
-      run: actionlint
+      glob: ".github/workflows/*.{yaml,yml}"
+      run: actionlint {staged_files}

data/lib/mihari/clients/crtsh.rb

@@ -12,7 +12,7 @@ module Mihari
       # @param [Integer, nil] timeout
       #
       def initialize(base_url = "https://crt.sh", headers: {}, timeout: nil)
-        super(base_url, headers:, timeout:)
+        super
       end
 
       #

data/lib/mihari/clients/dnstwister.rb

@@ -12,7 +12,7 @@ module Mihari
       # @param [Integer, nil] timeout
       #
       def initialize(base_url = "https://dnstwister.report", headers: {}, timeout: nil)
-        super(base_url, headers:, timeout:)
+        super
       end
 
       #

data/lib/mihari/clients/google_public_dns.rb

@@ -12,7 +12,7 @@ module Mihari
       # @param [Integer, nil] timeout
       #
       def initialize(base_url = "https://dns.google", headers: {}, timeout: nil)
-        super(base_url, headers:, timeout:)
+        super
       end
 
       #

data/lib/mihari/clients/mmdb.rb

@@ -12,7 +12,7 @@ module Mihari
       # @param [Integer, nil] timeout
       #
       def initialize(base_url = "https://ip.circl.lu", headers: {}, timeout: nil)
-        super(base_url, headers:, timeout:)
+        super
       end
 
       #

data/lib/mihari/clients/shodan_internet_db.rb

@@ -12,7 +12,7 @@ module Mihari
       # @param [Integer, nil] timeout
       #
       def initialize(base_url = "https://internetdb.shodan.io", headers: {}, timeout: nil)
-        super(base_url, headers:, timeout:)
+        super
       end
 
       #

data/lib/mihari/commands/rule.rb

@@ -26,14 +26,37 @@ module Mihari
               end
             end
 
-            desc "validate PATH", "Validate a rule"
+            desc "validate PATH", "Validate rule(s)"
             #
-            # Validate format of a rule
+            # Validate rule(s)
+            #
+            # @param [Array<String>] paths
+            #
+            def validate(*paths)
+              # @type [Array<Mihari::ValidationError>]
+              errors = paths.flat_map { |path| Dir.glob(path) }.map do |path|
+                Dry::Monads::Try[ValidationError] { Mihari::Rule.from_file(path) }
+              end.filter_map do |result|
+                result.exception if result.error?
+              end
+              return if errors.empty?
+
+              errors.each do |error|
+                data = Entities::ErrorMessage.represent(message: error.message, detail: error.detail)
+                warn JSON.pretty_generate(data.as_json)
+              end
+
+              exit 1
+            end
+
+            desc "format PATH", "format a rule"
+            #
+            # Format a rule file
             #
             # @param [String] path
             #
-            def validate(path)
-              rule = Dry::Monads::Try[ValidationError] { Mihari::Rule.from_yaml File.read(path) }.value!
+            def format(path)
+              rule = Dry::Monads::Try[ValidationError] { Mihari::Rule.from_file(path) }.value!
               puts rule.data.to_yaml
             end
 

data/lib/mihari/database.rb

@@ -6,7 +6,7 @@ ActiveSupport::Inflector.inflections(:en) { |inflect| inflect.acronym "CPE" }
 #
 # Mihari v7 DB schema
 #
-class V7Schema < ActiveRecord::Migration[7.1]
+class V7Schema < ActiveRecord::Migration[7.2]
   def change
     create_table :rules, id: :string, if_not_exists: true do |t|
       t.string :title, null: false

data/lib/mihari/enrichers/base.rb

@@ -10,7 +10,7 @@ module Mihari
       # @param [Hash, nil] options
       #
       def initialize(options: nil)
-        super(options:)
+        super
       end
 
       #

data/lib/mihari/enrichers/whois.rb

@@ -16,7 +16,9 @@ module Mihari
       def call(artifact)
         return if artifact.domain.nil?
 
-        artifact.whois_record ||= memoized_lookup(PublicSuffix.domain(artifact.domain))
+        artifact.tap do |tapped|
+          tapped.whois_record ||= memoized_lookup(PublicSuffix.domain(artifact.domain))
+        end
       end
 
       private

data/lib/mihari/models/artifact.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "ostruct"
+
 module Mihari
   module Models
     #
@@ -158,10 +160,7 @@ module Mihari
       # @return [Boolean] true if it is unique. Otherwise false.
       #
       def unique?(base_time: nil, artifact_ttl: nil)
-        artifact = self.class.joins(:alert).where(
-          data:,
-          alert: {rule_id:}
-        ).order(created_at: :desc).first
+        artifact = self.class.joins(:alert).where(data:, alert: {rule_id:}).order(created_at: :desc).first
         return true if artifact.nil?
 
         # check whether the artifact is decayed or not
@@ -179,7 +178,32 @@ module Mihari
       end
 
       def enrich
-        callable_enrichers.each { |enricher| enricher.result self }
+        enrich_by_enrichers callable_enrichers
+      end
+
+      #
+      # @param [Array<Mihari::Enrichers::Base>] enrichers
+      # @param [Boolean] parallel
+      #
+      # @return [Mihari::Models::Artifact]
+      #
+      def enrich_by_enrichers(enrichers)
+        # NOTE: doing parallel with ActiveRecord objects is troublesome (e.g. connection issue, etc.)
+        #       so converting the object to an OpenStruct object
+        s = struct
+        results = Parallel.map(enrichers) { |enricher| enricher.result s }
+        enriched = results.compact.map { |result| result.value_or(nil) }.compact
+
+        self.dns_records = enriched.map(&:dns_records).flatten.compact
+        self.cpes = enriched.map(&:cpes).flatten.compact
+        self.ports = enriched.map(&:ports).flatten.compact
+        self.vulnerabilities = enriched.map(&:vulnerabilities).flatten.compact
+
+        self.autonomous_system = enriched.map(&:autonomous_system).compact.first
+        self.geolocation = enriched.map(&:geolocation).compact.first
+        self.whois_record = enriched.map(&:whois_record).compact.first
+
+        self
       end
 
       #
@@ -195,6 +219,18 @@ module Mihari
         end
       end
 
+      def struct
+        OpenStruct.new(attributes).tap do |s|
+          s.domain = domain
+          s.cpes ||= []
+          s.dns_records ||= []
+          s.ports ||= []
+          s.reverse_dns_names ||= []
+          s.vulnerabilities ||= []
+          s.tags ||= []
+        end
+      end
+
       class << self
         # @!method search_by_filter(filter)
         #   @param [Mihari::Structs::Filters::Search] filter
@@ -212,7 +248,7 @@ module Mihari
       #
       def callable_enrichers
         @callable_enrichers ||= Mihari.enrichers.map(&:new).select do |enricher|
-          enricher.callable?(self)
+          enricher.callable? self
         end
       end
 

data/lib/mihari/rule.rb

@@ -5,6 +5,9 @@ module Mihari
     include Concerns::FalsePositiveNormalizable
     include Concerns::FalsePositiveValidatable
 
+    # @return [String, nil]
+    attr_reader :path_or_id
+
     # @return [Hash]
     attr_reader :data
 
@@ -19,9 +22,11 @@ module Mihari
     #
     # @param [Hash] data
     #
-    def initialize(**data)
+    # @param [Object] path_or_id
+    def initialize(path_or_id: nil, **data)
       super()
 
+      @path_or_id = path_or_id
       @data = data.deep_symbolize_keys
       @errors = nil
       @base_time = Time.now.utc
@@ -173,10 +178,7 @@ module Mihari
     #
     def enriched_artifacts
       @enriched_artifacts ||= Parallel.map(unique_artifacts) do |artifact|
-        artifact.tap do |tapped|
-          # NOTE: To apply changes correctly, enrichers should be applied to an artifact serially
-          enrichers.each { |enricher| enricher.result(tapped) }
-        end
+        artifact.enrich_by_enrichers enrichers
       end
     end
 
@@ -251,16 +253,29 @@ module Mihari
     end
 
     class << self
+      #
+      # Load rule from YAML file
+      #
+      # @param [String] path
+      #
+      # @return [Mihari::Rule]
+      #
+      def from_file(path)
+        yaml = File.read(path)
+        from_yaml(yaml, path: path)
+      end
+
       #
       # Load rule from YAML string
       #
       # @param [String] yaml
+      # @param [String, nil] path
       #
       # @return [Mihari::Rule]
       #
-      def from_yaml(yaml)
+      def from_yaml(yaml, path: nil)
         data = YAML.safe_load(ERB.new(yaml).result, permitted_classes: [Date, Symbol])
-        new(**data)
+        new(path_or_id: path, **data)
       end
 
       #
@@ -269,7 +284,7 @@ module Mihari
       # @return [Mihari::Rule]
       #
       def from_model(model)
-        new(**model.data)
+        new(path_or_id: model.id, **model.data)
       end
     end
 
@@ -409,7 +424,7 @@ module Mihari
       @data = result.to_h
       @errors = result.errors
 
-      raise ValidationError.new("Validation failed", errors) if errors?
+      raise ValidationError.new("#{path_or_id}: validation failed", errors) if errors?
     end
   end
 end

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "7.6.1"
+  VERSION = "7.6.3"
 end