mihari 7.1.2 → 7.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 52d1c8320fdb5233c9738f3b4599868260fef892599cdfb42da6c3af17583b75
-  data.tar.gz: 625cd92558eff5a4d5613e588cc5ee85b9b714a9af211788da1a294d8d54ac45
+  metadata.gz: 6b91d99562526b653b793f71e8ef5575f113d26859ca86b91f1980d1c44e898c
+  data.tar.gz: 07c302ce446b0f0986bfc82dd575ebde203f4b25b73a1aee72a2ce37a08b9b87
 SHA512:
-  metadata.gz: 96811d3ebfffcb27a7577b814be280916dd46bd3477233374ffafd2bf784d9e5133f89b1d7650c722c25e2f678f82380f5ced2eff8f9b7e1f3c96583f54a1114
-  data.tar.gz: 5c13581e670c7aff158e93335fe9c626294cad622f153e61a338f31bb5a1e459302f34a3a9ddf18e0bb322e1359c43a6b782387cdc800f9b047c04e5521f4c70
+  metadata.gz: 67d607a09ab2992b6721358b9e96e0c049a355221b2e97358440d70f96ef4933ac86b337bcc12bdc4b2aafccf4e5d6cf8cd68f4b2bbe567523bfba22b7fbd88c
+  data.tar.gz: 6f73de19824d31e21bae4ee7ce456c1b15fa0a875d1f07025c33ac0356ef257ac2eb819ffd8b685bf8914a472da04f691e5ae02a361397da9a83253d5345b108

data/Dockerfile

@@ -2,7 +2,7 @@ FROM ruby:3.2.2-alpine3.19
 
 ARG MIHARI_VERSION=0.0.0
 
-RUN apk --no-cache add build-base ruby-dev libpq-dev && \
+RUN apk --no-cache add build-base ruby-dev libpq-dev whois && \
   echo 'gem: --no-document' >> /usr/local/etc/gemrc && \
   gem install pg && \
   gem install mihari -v ${MIHARI_VERSION} && \

data/Rakefile

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "time"
+
 require "rspec/core/rake_task"
 require "standard/rake"
 
@@ -46,18 +48,24 @@ namespace :build do
   desc "Build Swagger doc"
   task :swagger, [:path] do |_t, args|
     args.with_defaults(path: "./frontend/swagger.yaml")
+
+    started_at = Time.now
     build_swagger_doc args.path
+    elapsed = (Time.now - started_at).floor(2)
+
+    puts "Swagger doc is built in #{elapsed}s"
   end
 end
 
-def ci?
-  ENV.fetch("CI", false)
-end
+task :build do
+  Rake::Task["build:swagger"].invoke
 
-unless ci?
-  task :build do
-    sh "./build_frontend.sh"
-  end
+  # Build ReDocs docs & frontend assets
+  sh "cd frontend && npm install && npm run docs && npm run build-only"
+  # Copy built assets into ./lib/web/public/
+  sh "rm -rf ./lib/mihari/web/public/"
+  sh "mkdir -p ./lib/mihari/web/public/"
+  sh "cp -r frontend/dist/* ./lib/mihari/web/public"
 end
 
 # require it later enables doing pre-build step (= build the frontend app)

data/build_frontend.sh

@@ -3,7 +3,7 @@
 CURRENT_DIR=${PWD}
 
 cd frontend
-npm ci
+npm install
 npm run build
 
 trash -r ${CURRENT_DIR}/lib/mihari/web/public/

data/lefthook.yml

@@ -1,5 +1,4 @@
 pre-commit:
-  parallel: true
   commands:
     standard:
       glob: "*.rb"
@@ -15,6 +14,10 @@ pre-commit:
       glob: "*.{js,ts,vue}"
       run: npx prettier --write {staged_files}
       stage_fixed: true
+    type-check:
+      root: "frontend/"
+      glob: "*.{js,ts,vue}"
+      run: npm run type-check
     actionlint:
       glob: ".github/workflows/*.yaml"
       run: actionlint

data/lib/mihari/actor.rb

@@ -53,10 +53,10 @@ module Mihari
     def validate_configuration!
       return if configured?
 
-      joined = self.class.configuration_keys.join(", ")
-      be = (self.class.configuration_keys.length > 1) ? "are" : "is"
-      message = "#{self.class.key} is not configured correctly. #{joined} #{be} missing."
-      raise ConfigurationError, message
+      message = "#{self.class.type.capitalize}:#{self.class.key} is not configured correctly"
+      detail = self.class.configuration_keys.map { |key| "#{key.upcase} is missing" }
+
+      raise ConfigurationError.new(message, detail)
     end
 
     def call(*args, **kwargs)
@@ -92,6 +92,23 @@ module Mihari
       def keys
         ([key] + [key_aliases]).flatten.compact.map(&:downcase)
       end
+
+      def configuration_keys
+        # Automatically generate configuration keys based on key
+        # For example,
+        # - Shodan analyzer's key is "shodan"
+        # - Mihari.config has "shodan_api_key"
+        # - Select "shodan_api_key" by using "#{key}_" prefix
+        Mihari.config.keys.select { |config_key| config_key.start_with?("#{key}_") }
+      end
+
+      def type
+        return "analyzer" if ancestors.include?(Mihari::Analyzers::Base)
+        return "emitter" if ancestors.include?(Mihari::Emitters::Base)
+        return "enricher" if ancestors.include?(Mihari::Enrichers::Base)
+
+        nil
+      end
     end
   end
 end

data/lib/mihari/analyzers/base.rb

@@ -63,12 +63,10 @@ module Mihari
         artifacts.compact.sort.map do |artifact|
           # No need to set data_type manually
           # It is set automatically in #initialize
-          artifact = artifact.is_a?(Models::Artifact) ? artifact : Models::Artifact.new(data: artifact)
-
-          artifact.source = self.class.key
-          artifact.query = query
-
-          artifact
+          (artifact.is_a?(Models::Artifact) ? artifact : Models::Artifact.new(data: artifact)).tap do |normalized|
+            normalized.source = self.class.key
+            normalized.query = query
+          end
         end.select(&:valid?).uniq(&:data)
       end
 
@@ -118,18 +116,9 @@ module Mihari
         #
         # @return [Mihari::Analyzers::Base]
         #
-        def from_query(params)
-          copied = params.deep_dup
-
-          # convert params into arguments for initialization
-          query = copied[:query]
-
-          # delete analyzer and query
-          %i[analyzer query].each { |key| copied.delete key }
-
-          copied[:options] = copied[:options] || nil
-
-          new(query, **copied)
+        def from_params(params)
+          query = params.delete(:query)
+          new(query, **params)
         end
 
         def inherited(child)

data/lib/mihari/analyzers/binaryedge.rb

@@ -24,12 +24,6 @@ module Mihari
         client.search_with_pagination(query, pagination_limit: pagination_limit).map(&:artifacts).flatten
       end
 
-      class << self
-        def configuration_keys
-          %w[binaryedge_api_key]
-        end
-      end
-
       private
 
       #

data/lib/mihari/analyzers/censys.rb

@@ -41,15 +41,6 @@ module Mihari
         configuration_keys? || (id? && secret?)
       end
 
-      class << self
-        #
-        # @return [Array<String>]
-        #
-        def configuration_keys
-          %w[censys_id censys_secret]
-        end
-      end
-
       private
 
       #

data/lib/mihari/analyzers/circl.rb

@@ -47,12 +47,6 @@ module Mihari
         configuration_keys? || (username? && password?)
       end
 
-      class << self
-        def configuration_keys
-          %w[circl_passive_password circl_passive_username]
-        end
-      end
-
       private
 
       def client

data/lib/mihari/analyzers/fofa.rb

@@ -35,12 +35,6 @@ module Mihari
         api_key? && email?
       end
 
-      class << self
-        def configuration_keys
-          %w[fofa_api_key fofa_email]
-        end
-      end
-
       private
 
       def email?

data/lib/mihari/analyzers/greynoise.rb

@@ -27,12 +27,6 @@ module Mihari
         ).map(&:artifacts).flatten
       end
 
-      class << self
-        def configuration_keys
-          %w[greynoise_api_key]
-        end
-      end
-
       private
 
       def client

data/lib/mihari/analyzers/hunterhow.rb

@@ -44,12 +44,6 @@ module Mihari
         end.flatten
       end
 
-      class << self
-        def configuration_keys
-          %w[hunterhow_api_key]
-        end
-      end
-
       private
 
       def client

data/lib/mihari/analyzers/onyphe.rb

@@ -29,12 +29,6 @@ module Mihari
         ).map(&:artifacts).flatten
       end
 
-      class << self
-        def configuration_keys
-          %w[onyphe_api_key]
-        end
-      end
-
       private
 
       def client

data/lib/mihari/analyzers/otx.rb

@@ -38,12 +38,6 @@ module Mihari
         end
       end
 
-      class << self
-        def configuration_keys
-          %w[otx_api_key]
-        end
-      end
-
       private
 
       def client

data/lib/mihari/analyzers/passivetotal.rb

@@ -50,10 +50,6 @@ module Mihari
       end
 
       class << self
-        def configuration_keys
-          %w[passivetotal_username passivetotal_api_key]
-        end
-
         #
         # @return [Array<String>, nil]
         #

data/lib/mihari/analyzers/pulsedive.rb

@@ -43,12 +43,6 @@ module Mihari
         end
       end
 
-      class << self
-        def configuration_keys
-          %w[pulsedive_api_key]
-        end
-      end
-
       private
 
       def client

data/lib/mihari/analyzers/securitytrails.rb

@@ -44,10 +44,6 @@ module Mihari
       end
 
       class << self
-        def configuration_keys
-          %w[securitytrails_api_key]
-        end
-
         #
         # @return [Array<String>, nil]
         #

data/lib/mihari/analyzers/shodan.rb

@@ -27,12 +27,6 @@ module Mihari
         ).map(&:artifacts).flatten.uniq(&:data)
       end
 
-      class << self
-        def configuration_keys
-          %w[shodan_api_key]
-        end
-      end
-
       private
 
       #

data/lib/mihari/analyzers/urlscan.rb

@@ -37,12 +37,6 @@ module Mihari
         artifacts.select { |artifact| allowed_data_types.include? artifact.data_type }
       end
 
-      class << self
-        def configuration_keys
-          %w[urlscan_api_key]
-        end
-      end
-
       private
 
       def client

data/lib/mihari/analyzers/virustotal.rb

@@ -39,10 +39,6 @@ module Mihari
       end
 
       class << self
-        def configuration_keys
-          %w[virustotal_api_key]
-        end
-
         #
         # @return [Array<String>, nil]
         #

data/lib/mihari/analyzers/virustotal_intelligence.rb

@@ -24,12 +24,6 @@ module Mihari
         client.intel_search_with_pagination(query, pagination_limit: pagination_limit).map(&:artifacts).flatten
       end
 
-      class << self
-        def configuration_keys
-          %w[virustotal_api_key]
-        end
-      end
-
       class << self
         #
         # @return [String]
@@ -44,6 +38,13 @@ module Mihari
         def key_aliases
           ["vt_intel"]
         end
+
+        #
+        # @return [Array<String>]
+        #
+        def configuration_keys
+          %w[virustotal_api_key]
+        end
       end
 
       private

data/lib/mihari/analyzers/zoomeye.rb

@@ -40,12 +40,6 @@ module Mihari
         end
       end
 
-      class << self
-        def configuration_keys
-          %w[zoomeye_api_key]
-        end
-      end
-
       private
 
       #

data/lib/mihari/commands/web.rb

@@ -10,10 +10,10 @@ module Mihari
         def included(thor)
           thor.class_eval do
             desc "web", "Start the web app"
-            method_option :port, type: :numeric, default: 9292, desc: "Hostname to listen on"
-            method_option :host, type: :string, default: "localhost", desc: "Port to listen on"
-            method_option :threads, type: :string, default: "0:5", desc: "min:max threads to use"
-            method_option :verbose, type: :boolean, default: true, desc: "Report each request"
+            method_option :port, type: :numeric, default: 9292, desc: "Port to listen on"
+            method_option :host, type: :string, default: "localhost", desc: "Hostname to listen on"
+            method_option :threads, type: :string, default: "0:3", desc: "min:max threads to use"
+            method_option :verbose, type: :boolean, default: false, desc: "Don't report each request"
             method_option :worker_timeout, type: :numeric, default: 60, desc: "Worker timeout value (in seconds)"
             method_option :open, type: :boolean, default: true, desc: "Whether to open the app in browser or not"
             method_option :env, type: :string, default: "production", desc: "Environment"

data/lib/mihari/concerns/falsepositive_normalizable.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Concerns
+    #
+    # False positive normalizable concern
+    #
+    module FalsePositiveNormalizable
+      extend ActiveSupport::Concern
+
+      prepend MemoWise
+
+      #
+      # Normalize a falsepositive value
+      #
+      # @param [String] value
+      #
+      # @return [String, Regexp]
+      #
+      def normalize_falsepositive(value)
+        return value if !value.start_with?("/") || !value.end_with?("/")
+
+        # if a value is surrounded by slashes, take it as a regexp
+        value_without_slashes = value[1..-2]
+        Regexp.compile value_without_slashes.to_s
+      end
+      memo_wise :normalize_falsepositive
+    end
+  end
+end

data/lib/mihari/concerns/falsepositive_validatable.rb

@@ -8,23 +8,7 @@ module Mihari
     module FalsePositiveValidatable
       extend ActiveSupport::Concern
 
-      prepend MemoWise
-
-      #
-      # Normalize a falsepositive value
-      #
-      # @param [String] value
-      #
-      # @return [String, Regexp]
-      #
-      def normalize_falsepositive(value)
-        return value if !value.start_with?("/") || !value.end_with?("/")
-
-        # if a value is surrounded by slashes, take it as a regexp
-        value_without_slashes = value[1..-2]
-        Regexp.compile value_without_slashes.to_s
-      end
-      memo_wise :normalize_falsepositive
+      include FalsePositiveNormalizable
 
       #
       # Check whether a value is valid format as a disallowed data value

data/lib/mihari/config.rb

@@ -170,7 +170,7 @@ module Mihari
     # @return [Array<String>]
     #
     def keys
-      to_h.keys.map(&:to_s).map(&:upcase)
+      @keys ||= to_h.keys.map(&:to_s).map(&:downcase)
     end
   end
 end

data/lib/mihari/database.rb

@@ -104,11 +104,28 @@ class V7Schema < ActiveRecord::Migration[7.1]
   end
 end
 
+class V72Schema < ActiveRecord::Migration[7.1]
+  def change
+    create_table :vulnerabilities, if_not_exists: true do |t|
+      t.string :name, null: false
+      t.datetime :created_at
+
+      t.belongs_to :artifact, foreign_key: true, null: false
+    end
+
+    rename_column :cpes, :cpe, :name if ActiveRecord::Base.connection.column_exists?(:cpes, :cpe)
+    rename_column :autonomous_systems, :asn, :number if ActiveRecord::Base.connection.column_exists?(
+      :autonomous_systems, :asn
+    )
+    rename_column :ports, :port, :number if ActiveRecord::Base.connection.column_exists?(:ports, :port)
+  end
+end
+
 #
 # @return [Array<ActiveRecord::Migration>] schemas
 #
 def schemas
-  [V7Schema]
+  [V7Schema, V72Schema]
 end
 
 module Mihari

data/lib/mihari/emitters/database.rb

@@ -24,12 +24,6 @@ module Mihari
       def target
         Mihari.config.database_url.host || Mihari.config.database_url.to_s
       end
-
-      class << self
-        def configuration_keys
-          %w[database_url]
-        end
-      end
     end
   end
 end

data/lib/mihari/emitters/misp.rb

@@ -63,12 +63,6 @@ module Mihari
         URI(url).host || "N/A"
       end
 
-      class << self
-        def configuration_keys
-          %w[misp_url misp_api_key]
-        end
-      end
-
       private
 
       def client

data/lib/mihari/emitters/slack.rb

@@ -176,21 +176,11 @@ module Mihari
       # @return [::Slack::Notifier]
       #
       def notifier
-        @notifier ||= [].tap do |out|
-          out << if timeout.nil?
-            ::Slack::Notifier.new(
-              webhook_url,
-              channel: channel, username: username
-            )
-          else
-            ::Slack::Notifier.new(
-              webhook_url,
-              channel: channel,
-              username: username,
-              http_options: { timeout: timeout }
-            )
-          end
-        end.first
+        @notifier ||= lambda do
+          return ::Slack::Notifier.new(webhook_url, channel: channel, username: username) if timeout.nil?
+
+          ::Slack::Notifier.new(webhook_url, channel: channel, username: username, http_options: { timeout: timeout })
+        end.call
       end
 
       #
@@ -227,12 +217,6 @@ module Mihari
 
         notifier.post(text: text, attachments: attachments, mrkdwn: true)
       end
-
-      class << self
-        def configuration_keys
-          %w[slack_webhook_url slack_channel]
-        end
-      end
     end
   end
 end

data/lib/mihari/emitters/the_hive.rb

@@ -53,12 +53,6 @@ module Mihari
         client.alert payload
       end
 
-      class << self
-        def configuration_keys
-          %w[thehive_url thehive_api_key]
-        end
-      end
-
       private
 
       def client

data/lib/mihari/enrichers/whois.rb

@@ -53,13 +53,11 @@ module Mihari
       # @return [::Whois::Client]
       #
       def whois
-        @whois ||= [].tap do |out|
-          out << if timeout.nil?
-            ::Whois::Client.new
-          else
-            ::Whois::Client.new(timeout: timeout)
-          end
-        end.last
+        @whois ||= lambda do
+          return ::Whois::Client.new if timeout.nil?
+
+          ::Whois::Client.new(timeout: timeout)
+        end.call
       end
 
       #

data/lib/mihari/entities/artifact.rb

@@ -8,12 +8,12 @@ module Mihari
       expose :data_type, documentation: { type: String, required: true }, as: :dataType
       expose :source, documentation: { type: String, required: true }
       expose :query, documentation: { type: String, required: false }
-      expose :metadata, documentation: { type: Hash }
       expose :created_at, documentation: { type: DateTime, required: true }, as: :createdAt
+      expose :tags, using: Entities::Tag, documentation: { type: Entities::Tag, is_array: true, required: true }
     end
 
     class Artifact < BaseArtifact
-      expose :tags, using: Entities::Tag, documentation: { type: Entities::Tag, is_array: true, required: true }
+      expose :metadata, documentation: { type: Hash }
       expose :autonomous_system, using: Entities::AutonomousSystem,
         documentation: { type: Entities::AutonomousSystem, required: false }, as: :autonomousSystem
       expose :geolocation, using: Entities::Geolocation, documentation: { type: Entities::Geolocation, required: false }
@@ -36,6 +36,10 @@ module Mihari
         as: :ports do |status, _options|
         status.ports.empty? ? nil : status.ports
       end
+      expose :vulnerabilities, using: Vulnerability, documentation: { type: Vulnerability, is_array: true, required: false },
+        as: :vulnerabilities do |status, _options|
+        status.vulnerabilities.empty? ? nil : status.vulnerabilities
+      end
     end
 
     class ArtifactsWithPagination < Pagination

data/lib/mihari/entities/autonomous_system.rb

@@ -3,7 +3,7 @@
 module Mihari
   module Entities
     class AutonomousSystem < Grape::Entity
-      expose :asn, documentation: { type: Integer, required: true }
+      expose :number, documentation: { type: Integer, required: true }
     end
   end
 end

data/lib/mihari/entities/cpe.rb

@@ -3,7 +3,7 @@
 module Mihari
   module Entities
     class CPE < Grape::Entity
-      expose :cpe, documentation: { type: String, required: true }
+      expose :name, documentation: { type: String, required: true }
       expose :created_at, documentation: { type: DateTime, required: true }, as: :createdAt
     end
   end