mihari 7.1.3 → 7.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d7463506c12a2476cfea5dda939e88e01150fb6539c232e102c15fca4c57ab99
-  data.tar.gz: fdb700f461140badc1d90b3199999a97e483e57ae9c9bfed2d0157a94f35529d
+  metadata.gz: 328a34edf637c36456cc7de39fccabdaaf75937b449b5e8a8e0434e71b9328c2
+  data.tar.gz: 8483c669cfb3e715c86b4a3878961885e4cd4f93611e487c4360cb3030267c18
 SHA512:
-  metadata.gz: 18d9dd1f63056cd2731caa823904fd9df80ca21e63a540eab89d06e7951fa10cde498344049f3ba7de8740da2e8cfb9da9fe660f77c3a1abca5e308e36a6973c
-  data.tar.gz: 5a8ce97455cb0e68538af1ca2c8af5b89ea96f817d4f11cf6c00851035a20288f9bc2d2de16b58b52f3e72e988c77a2e4d8744a3fbebfd27c2cb6895d4c4dccb
+  metadata.gz: 9858608c1ceb30f846a27b487e4bcbbad463aba56a401991fdade512bf6a7a7f028e5facecd764b200d94795a1da1a2b631b85540dfe1bbf002c756100b6627f
+  data.tar.gz: c050fdafab0e7855eb610ad3d50762583ca931d31afeeb57fc654a2dc65ff381f20cb31eb28d0a61b5888c0a19df02320b09a529830fe332389b59f739721aba

data/Dockerfile

@@ -1,8 +1,8 @@
-FROM ruby:3.2.2-alpine3.19
+FROM ruby:3.3.0-alpine3.19
 
 ARG MIHARI_VERSION=0.0.0
 
-RUN apk --no-cache add build-base ruby-dev libpq-dev && \
+RUN apk --no-cache add build-base ruby-dev libpq-dev whois && \
   echo 'gem: --no-document' >> /usr/local/etc/gemrc && \
   gem install pg && \
   gem install mihari -v ${MIHARI_VERSION} && \

data/Rakefile

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "time"
+
 require "rspec/core/rake_task"
 require "standard/rake"
 
@@ -46,13 +48,18 @@ namespace :build do
   desc "Build Swagger doc"
   task :swagger, [:path] do |_t, args|
     args.with_defaults(path: "./frontend/swagger.yaml")
+
+    started_at = Time.now
     build_swagger_doc args.path
+    elapsed = (Time.now - started_at).floor(2)
+
+    puts "Swagger doc is built in #{elapsed}s"
   end
 end
 
 task :build do
-  # Build Swagger dos
   Rake::Task["build:swagger"].invoke
+
   # Build ReDocs docs & frontend assets
   sh "cd frontend && npm install && npm run docs && npm run build-only"
   # Copy built assets into ./lib/web/public/

data/lefthook.yml

@@ -1,5 +1,4 @@
 pre-commit:
-  parallel: true
   commands:
     standard:
       glob: "*.rb"
@@ -15,6 +14,10 @@ pre-commit:
       glob: "*.{js,ts,vue}"
       run: npx prettier --write {staged_files}
       stage_fixed: true
+    type-check:
+      root: "frontend/"
+      glob: "*.{js,ts,vue}"
+      run: npm run type-check
     actionlint:
       glob: ".github/workflows/*.yaml"
       run: actionlint

data/lib/mihari/actor.rb

@@ -50,6 +50,13 @@ module Mihari
       options[:timeout]
     end
 
+    #
+    # @return [Boolean]
+    #
+    def parallel?
+      options[:parallel] || Mihari.config.parallel
+    end
+
     def validate_configuration!
       return if configured?
 
@@ -93,6 +100,15 @@ module Mihari
         ([key] + [key_aliases]).flatten.compact.map(&:downcase)
       end
 
+      def configuration_keys
+        # Automatically generate configuration keys based on key
+        # For example,
+        # - Shodan analyzer's key is "shodan"
+        # - Mihari.config has "shodan_api_key"
+        # - Select "shodan_api_key" by using "#{key}_" prefix
+        Mihari.config.keys.select { |config_key| config_key.start_with?("#{key}_") }
+      end
+
       def type
         return "analyzer" if ancestors.include?(Mihari::Analyzers::Base)
         return "emitter" if ancestors.include?(Mihari::Emitters::Base)

data/lib/mihari/analyzers/base.rb

@@ -40,13 +40,6 @@ module Mihari
         options[:ignore_error] || Mihari.config.ignore_error
       end
 
-      #
-      # @return [Boolean]
-      #
-      def parallel?
-        options[:parallel] || Mihari.config.parallel
-      end
-
       # @return [Array<String>, Array<Mihari::Models::Artifact>]
       def artifacts
         raise NotImplementedError, "You must implement #{self.class}##{__method__}"
@@ -63,12 +56,10 @@ module Mihari
         artifacts.compact.sort.map do |artifact|
           # No need to set data_type manually
           # It is set automatically in #initialize
-          artifact = artifact.is_a?(Models::Artifact) ? artifact : Models::Artifact.new(data: artifact)
-
-          artifact.source = self.class.key
-          artifact.query = query
-
-          artifact
+          (artifact.is_a?(Models::Artifact) ? artifact : Models::Artifact.new(data: artifact)).tap do |normalized|
+            normalized.source = self.class.key
+            normalized.query = query
+          end
         end.select(&:valid?).uniq(&:data)
       end
 
@@ -118,18 +109,9 @@ module Mihari
         #
         # @return [Mihari::Analyzers::Base]
         #
-        def from_query(params)
-          copied = params.deep_dup
-
-          # convert params into arguments for initialization
-          query = copied[:query]
-
-          # delete analyzer and query
-          %i[analyzer query].each { |key| copied.delete key }
-
-          copied[:options] = copied[:options] || nil
-
-          new(query, **copied)
+        def from_params(params)
+          query = params.delete(:query)
+          new(query, **params)
         end
 
         def inherited(child)

data/lib/mihari/analyzers/binaryedge.rb

@@ -24,12 +24,6 @@ module Mihari
         client.search_with_pagination(query, pagination_limit: pagination_limit).map(&:artifacts).flatten
       end
 
-      class << self
-        def configuration_keys
-          %w[binaryedge_api_key]
-        end
-      end
-
       private
 
       #

data/lib/mihari/analyzers/censys.rb

@@ -41,15 +41,6 @@ module Mihari
         configuration_keys? || (id? && secret?)
       end
 
-      class << self
-        #
-        # @return [Array<String>]
-        #
-        def configuration_keys
-          %w[censys_id censys_secret]
-        end
-      end
-
       private
 
       #

data/lib/mihari/analyzers/circl.rb

@@ -47,12 +47,6 @@ module Mihari
         configuration_keys? || (username? && password?)
       end
 
-      class << self
-        def configuration_keys
-          %w[circl_passive_password circl_passive_username]
-        end
-      end
-
       private
 
       def client

data/lib/mihari/analyzers/fofa.rb

@@ -35,12 +35,6 @@ module Mihari
         api_key? && email?
       end
 
-      class << self
-        def configuration_keys
-          %w[fofa_api_key fofa_email]
-        end
-      end
-
       private
 
       def email?

data/lib/mihari/analyzers/greynoise.rb

@@ -27,12 +27,6 @@ module Mihari
         ).map(&:artifacts).flatten
       end
 
-      class << self
-        def configuration_keys
-          %w[greynoise_api_key]
-        end
-      end
-
       private
 
       def client

data/lib/mihari/analyzers/hunterhow.rb

@@ -44,12 +44,6 @@ module Mihari
         end.flatten
       end
 
-      class << self
-        def configuration_keys
-          %w[hunterhow_api_key]
-        end
-      end
-
       private
 
       def client

data/lib/mihari/analyzers/onyphe.rb

@@ -29,12 +29,6 @@ module Mihari
         ).map(&:artifacts).flatten
       end
 
-      class << self
-        def configuration_keys
-          %w[onyphe_api_key]
-        end
-      end
-
       private
 
       def client

data/lib/mihari/analyzers/otx.rb

@@ -38,12 +38,6 @@ module Mihari
         end
       end
 
-      class << self
-        def configuration_keys
-          %w[otx_api_key]
-        end
-      end
-
       private
 
       def client

data/lib/mihari/analyzers/passivetotal.rb

@@ -50,10 +50,6 @@ module Mihari
       end
 
       class << self
-        def configuration_keys
-          %w[passivetotal_username passivetotal_api_key]
-        end
-
         #
         # @return [Array<String>, nil]
         #

data/lib/mihari/analyzers/pulsedive.rb

@@ -43,12 +43,6 @@ module Mihari
         end
       end
 
-      class << self
-        def configuration_keys
-          %w[pulsedive_api_key]
-        end
-      end
-
       private
 
       def client

data/lib/mihari/analyzers/securitytrails.rb

@@ -44,10 +44,6 @@ module Mihari
       end
 
       class << self
-        def configuration_keys
-          %w[securitytrails_api_key]
-        end
-
         #
         # @return [Array<String>, nil]
         #

data/lib/mihari/analyzers/shodan.rb

@@ -27,12 +27,6 @@ module Mihari
         ).map(&:artifacts).flatten.uniq(&:data)
       end
 
-      class << self
-        def configuration_keys
-          %w[shodan_api_key]
-        end
-      end
-
       private
 
       #

data/lib/mihari/analyzers/urlscan.rb

@@ -37,12 +37,6 @@ module Mihari
         artifacts.select { |artifact| allowed_data_types.include? artifact.data_type }
       end
 
-      class << self
-        def configuration_keys
-          %w[urlscan_api_key]
-        end
-      end
-
       private
 
       def client

data/lib/mihari/analyzers/virustotal.rb

@@ -39,10 +39,6 @@ module Mihari
       end
 
       class << self
-        def configuration_keys
-          %w[virustotal_api_key]
-        end
-
         #
         # @return [Array<String>, nil]
         #

data/lib/mihari/analyzers/virustotal_intelligence.rb

@@ -24,12 +24,6 @@ module Mihari
         client.intel_search_with_pagination(query, pagination_limit: pagination_limit).map(&:artifacts).flatten
       end
 
-      class << self
-        def configuration_keys
-          %w[virustotal_api_key]
-        end
-      end
-
       class << self
         #
         # @return [String]
@@ -44,6 +38,13 @@ module Mihari
         def key_aliases
           ["vt_intel"]
         end
+
+        #
+        # @return [Array<String>]
+        #
+        def configuration_keys
+          %w[virustotal_api_key]
+        end
       end
 
       private

data/lib/mihari/analyzers/zoomeye.rb

@@ -40,12 +40,6 @@ module Mihari
         end
       end
 
-      class << self
-        def configuration_keys
-          %w[zoomeye_api_key]
-        end
-      end
-
       private
 
       #

data/lib/mihari/commands/web.rb

@@ -12,7 +12,7 @@ module Mihari
             desc "web", "Start the web app"
             method_option :port, type: :numeric, default: 9292, desc: "Port to listen on"
             method_option :host, type: :string, default: "localhost", desc: "Hostname to listen on"
-            method_option :threads, type: :string, default: "0:5", desc: "min:max threads to use"
+            method_option :threads, type: :string, default: "0:3", desc: "min:max threads to use"
             method_option :verbose, type: :boolean, default: false, desc: "Don't report each request"
             method_option :worker_timeout, type: :numeric, default: 60, desc: "Worker timeout value (in seconds)"
             method_option :open, type: :boolean, default: true, desc: "Whether to open the app in browser or not"

data/lib/mihari/concerns/falsepositive_normalizable.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Concerns
+    #
+    # False positive normalizable concern
+    #
+    module FalsePositiveNormalizable
+      extend ActiveSupport::Concern
+
+      prepend MemoWise
+
+      #
+      # Normalize a falsepositive value
+      #
+      # @param [String] value
+      #
+      # @return [String, Regexp]
+      #
+      def normalize_falsepositive(value)
+        return value if !value.start_with?("/") || !value.end_with?("/")
+
+        # if a value is surrounded by slashes, take it as a regexp
+        value_without_slashes = value[1..-2]
+        Regexp.compile value_without_slashes.to_s
+      end
+      memo_wise :normalize_falsepositive
+    end
+  end
+end

data/lib/mihari/concerns/falsepositive_validatable.rb

@@ -8,23 +8,7 @@ module Mihari
     module FalsePositiveValidatable
       extend ActiveSupport::Concern
 
-      prepend MemoWise
-
-      #
-      # Normalize a falsepositive value
-      #
-      # @param [String] value
-      #
-      # @return [String, Regexp]
-      #
-      def normalize_falsepositive(value)
-        return value if !value.start_with?("/") || !value.end_with?("/")
-
-        # if a value is surrounded by slashes, take it as a regexp
-        value_without_slashes = value[1..-2]
-        Regexp.compile value_without_slashes.to_s
-      end
-      memo_wise :normalize_falsepositive
+      include FalsePositiveNormalizable
 
       #
       # Check whether a value is valid format as a disallowed data value

data/lib/mihari/config.rb

@@ -170,7 +170,7 @@ module Mihari
     # @return [Array<String>]
     #
     def keys
-      to_h.keys.map(&:to_s).map(&:upcase)
+      @keys ||= to_h.keys.map(&:to_s).map(&:downcase)
     end
   end
 end

data/lib/mihari/database.rb

@@ -104,11 +104,28 @@ class V7Schema < ActiveRecord::Migration[7.1]
   end
 end
 
+class V72Schema < ActiveRecord::Migration[7.1]
+  def change
+    create_table :vulnerabilities, if_not_exists: true do |t|
+      t.string :name, null: false
+      t.datetime :created_at
+
+      t.belongs_to :artifact, foreign_key: true, null: false
+    end
+
+    rename_column :cpes, :cpe, :name if ActiveRecord::Base.connection.column_exists?(:cpes, :cpe)
+    rename_column :autonomous_systems, :asn, :number if ActiveRecord::Base.connection.column_exists?(
+      :autonomous_systems, :asn
+    )
+    rename_column :ports, :port, :number if ActiveRecord::Base.connection.column_exists?(:ports, :port)
+  end
+end
+
 #
 # @return [Array<ActiveRecord::Migration>] schemas
 #
 def schemas
-  [V7Schema]
+  [V7Schema, V72Schema]
 end
 
 module Mihari

data/lib/mihari/emitters/database.rb

@@ -24,12 +24,6 @@ module Mihari
       def target
         Mihari.config.database_url.host || Mihari.config.database_url.to_s
       end
-
-      class << self
-        def configuration_keys
-          %w[database_url]
-        end
-      end
     end
   end
 end

data/lib/mihari/emitters/misp.rb

@@ -63,12 +63,6 @@ module Mihari
         URI(url).host || "N/A"
       end
 
-      class << self
-        def configuration_keys
-          %w[misp_url misp_api_key]
-        end
-      end
-
       private
 
       def client

data/lib/mihari/emitters/slack.rb

@@ -176,21 +176,11 @@ module Mihari
       # @return [::Slack::Notifier]
       #
       def notifier
-        @notifier ||= [].tap do |out|
-          out << if timeout.nil?
-            ::Slack::Notifier.new(
-              webhook_url,
-              channel: channel, username: username
-            )
-          else
-            ::Slack::Notifier.new(
-              webhook_url,
-              channel: channel,
-              username: username,
-              http_options: { timeout: timeout }
-            )
-          end
-        end.first
+        @notifier ||= lambda do
+          return ::Slack::Notifier.new(webhook_url, channel: channel, username: username) if timeout.nil?
+
+          ::Slack::Notifier.new(webhook_url, channel: channel, username: username, http_options: { timeout: timeout })
+        end.call
       end
 
       #
@@ -227,12 +217,6 @@ module Mihari
 
         notifier.post(text: text, attachments: attachments, mrkdwn: true)
       end
-
-      class << self
-        def configuration_keys
-          %w[slack_webhook_url slack_channel]
-        end
-      end
     end
   end
 end

data/lib/mihari/emitters/the_hive.rb

@@ -53,12 +53,6 @@ module Mihari
         client.alert payload
       end
 
-      class << self
-        def configuration_keys
-          %w[thehive_url thehive_api_key]
-        end
-      end
-
       private
 
       def client

data/lib/mihari/enrichers/base.rb

@@ -6,46 +6,88 @@ module Mihari
     # Base class for enrichers
     #
     class Base < Actor
-      prepend MemoWise
-
+      #
+      # @param [Hash, nil] options
+      #
       def initialize(options: nil)
         super(options: options)
       end
 
       #
-      # @param [String] value
+      # Enrich an artifact
+      #
+      # @param [Mihari::Models::Artifact] artifact
       #
-      def call(value)
+      # @return [Mihari::Models::Artifact]
+      #
+      def call(artifact)
         raise NotImplementedError, "You must implement #{self.class}##{__method__}"
       end
 
       #
-      # @param [Mihari::Models::Artifact] value
+      # @param [Mihari::Models::Artifact] artifact
       #
       # @return [Dry::Monads::Result::Success<Object>, Dry::Monads::Result::Failure]
       #
-      def result(value)
+      def result(artifact)
+        return unless callable?(artifact)
+
         result = Try[StandardError] do
-          retry_on_error(
-            times: retry_times,
-            interval: retry_interval,
-            exponential_backoff: retry_exponential_backoff
-          ) { call value }
+          retry_on_error(times: retry_times, interval: retry_interval,
+            exponential_backoff: retry_exponential_backoff) do
+            call artifact
+          end
         end.to_result
 
         if result.failure?
-          Mihari.logger.warn("Enricher:#{self.class.key} for #{value.truncate(32)} failed: #{result.failure}")
+          Mihari.logger.warn("Enricher:#{self.class.key} for #{artifact.data.truncate(32)} failed: #{result.failure}")
         end
 
         result
       end
 
+      #
+      # @param [Mihari::Models::Artifact] artifact
+      #
+      # @return [Boolean]
+      #
+      def callable?(artifact)
+        callable_data_type?(artifact) && callable_relationships?(artifact)
+      end
+
       class << self
         def inherited(child)
           super
           Mihari.enrichers << child
         end
       end
+
+      private
+
+      #
+      # @param [Mihari::Models::Artifact] artifact
+      #
+      # @return [Boolean]
+      #
+      def callable_data_type?(artifact)
+        supported_data_types.include? artifact.data_type
+      end
+
+      #
+      # @param [Mihari::Models::Artifact] artifact
+      #
+      # @return [Boolean]
+      #
+      def callable_relationships?(artifact)
+        raise NotImplementedError, "You must implement #{self.class}##{__method__}"
+      end
+
+      #
+      # @return [Array<String>]
+      #
+      def supported_data_types
+        []
+      end
     end
   end
 end