mihari 6.3.0 → 7.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f027c5c24759f4e09d5dad277df4cad58a1705478eba1f3b9b6b354a896a29a5
-  data.tar.gz: a908c432c313bab4f4af70b0b32fb1f2e65c9f40926dc3099c4869330080005c
+  metadata.gz: bc368e612c78350f6b66f4fdcf815a63f4a292e185a2348dce8110b7d17e1692
+  data.tar.gz: b3e9939de4ee9488bb6b48793cf5ebe196f50e92eb8f6c1eadb3ed0b201fd2c0
 SHA512:
-  metadata.gz: 8110df87f854b6e06ea8477c94c6563b2450dae4585738f256da9c94203d14c6fb3ce11e775a470dae04d4d3934549ae71068e6b3f3d3eca48ea3689d87a4cd4
-  data.tar.gz: 375855d7dba59d13ff894472f5f6d5ccc82f36de73aeb12c4144c05863c64283f2ff451dd4531643ba3ddbb218181937d57da1a42bb28970f78170e3fae5dfbf
+  metadata.gz: 217918561a441c15d4c55e12fa90792230dee66186ad12db8629c30ec848a2a4711eb154d1a008f153abadb3d9cc97ad43ce574d175d167873f3b7c44f0de011
+  data.tar.gz: 2d16db3598a3d5918499ce7119f7e69a548f0891dd6f9c5ee5764508244486c6ba255e190f65cca79559db888936b7975db86eadefb7e548b476fb3925316f32

data/.gitignore

@@ -52,20 +52,14 @@ Gemfile.lock
 # rspec
 .rspec_status
 
-# solargraph
-.solargraph.yml
-
 # SQLite
 *.db
 *.db-shm
 *.db-wal
 
-# Config
-mihari.yml
-
-# Rule
-rule.yml
-!lib/mihari/templates/rule.yml
-
 # Frontend assets
 lib/mihari/web/public/
+
+# Rules & Alerts
+rules/
+alerts/

data/.rubocop.yml

@@ -18,5 +18,7 @@ RSpec/MultipleMemoizedHelpers:
 RSpec/ExampleLength:
   Max: 20
 require:
+  - rubocop-factory_bot
+  - rubocop-rake
   - rubocop-rspec
   - rubocop-yard

data/Dockerfile

@@ -0,0 +1,14 @@
+FROM ruby:3.2.2-alpine3.19
+
+ARG MIHARI_VERSION=0.0.0
+
+RUN apk --no-cache add build-base ruby-dev libpq-dev && \
+  echo 'gem: --no-document' >> /usr/local/etc/gemrc && \
+  gem install pg && \
+  gem install mihari -v ${MIHARI_VERSION} && \
+  apk del --purge build-base ruby-dev && \
+  rm -rf /usr/local/bundle/cache/*
+
+ENTRYPOINT ["mihari"]
+
+CMD ["--help"]

data/config.ru

@@ -1,8 +1,10 @@
-require "./lib/mihari"
+$LOAD_PATH.unshift("#{__dir__}/../lib")
+
+require "mihari"
+require "mihari/web/application"
 
 require "better_errors"
 
-# set rack env as development
-ENV["RACK_ENV"] ||= "development"
+ENV["APP_ENV"] ||= "development"
 
 run Mihari::Web::App.instance

data/docker-compose.yml

@@ -0,0 +1,61 @@
+version: "3"
+services:
+  database:
+    image: postgres:16
+    volumes:
+      - postgresql:/var/lib/postgresql/data
+    ports:
+      - ${POSTGRES_PORT:-5432}:5432
+    environment:
+      - POSTGRES_USER=${POSTGRES_USER:-user}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-password}
+      - POSTGRES_DB=${POSTGRES_DB:-mihari}
+    restart: always
+
+  redis:
+    image: "redis/redis-stack:6.2.6-v10"
+    restart: always
+    ports:
+      - ${REDIS_PORT:-6379}:6379
+      - ${REDIS_INSIGHT_PORT:-8001}:8001
+    volumes:
+      - redis:/data
+
+  mihari-init:
+    image: ghcr.io/ninoseki/mihari:latest
+    environment:
+      - DATABASE_URL=${DATABASE_URL:-postgresql://${POSTGRES_USER:-user}:${POSTGRES_PASSWORD:-password}@database:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-mihari}}
+    entrypoint: ["mihari", "db", "migrate"]
+    depends_on:
+      - database
+
+  mihari:
+    image: ghcr.io/ninoseki/mihari:latest
+    ports:
+      - ${MIHARI_PORT:-9292}:9292
+    environment:
+      - DATABASE_URL=${DATABASE_URL:-postgresql://${POSTGRES_USER:-user}:${POSTGRES_PASSWORD:-password}@database:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-mihari}}
+      - REDIS_URL=${REDIS_URL:-redis://redis:${REDIS_PORT:-6379}}
+    env_file:
+      - .env
+    entrypoint: ["mihari", "web", "--host", "0.0.0.0"]
+    restart: always
+    depends_on:
+      - mihari-init
+
+  sidekiq:
+    image: ghcr.io/ninoseki/mihari:latest
+    environment:
+      - DATABASE_URL=${DATABASE_URL:-postgresql://${POSTGRES_USER:-user}:${POSTGRES_PASSWORD:-password}@database:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-mihari}}
+      - SIDEKIQ_REDIS_URL=${REDIS_URL:-redis://redis:${REDIS_PORT:-6379}}
+    env_file:
+      - .env
+    entrypoint: ["mihari", "sidekiq"]
+    restart: always
+    depends_on:
+      - mihari-init
+      - redis
+
+volumes:
+  postgresql:
+  redis:

data/exe/mihari

@@ -4,5 +4,6 @@
 $LOAD_PATH.unshift("#{__dir__}/../lib")
 
 require "mihari"
+require "mihari/cli/application"
 
-Mihari::CLI::Main.start
+Mihari::CLI::App.start

data/lefthook.yml

@@ -10,3 +10,11 @@ pre-commit:
       glob: "*.{js,ts,vue}"
       run: npx eslint --fix {staged_files}
       stage_fixed: true
+    prettier:
+      root: "frontend/"
+      glob: "*.{js,ts,vue}"
+      run: npx prettier --write {staged_files}
+      stage_fixed: true
+    actionlint:
+      glob: ".github/workflows/*.yaml"
+      run: actionlint

data/lib/mihari/actor.rb

@@ -7,8 +7,8 @@ module Mihari
   class Actor
     include Dry::Monads[:result, :try]
 
-    include Mixins::Configurable
-    include Mixins::Retriable
+    include Concerns::Configurable
+    include Concerns::Retriable
 
     # @return [Hash]
     attr_reader :options
@@ -53,8 +53,8 @@ module Mihari
     def validate_configuration!
       return if configured?
 
-      joined = configuration_keys.join(", ")
-      be = (configuration_keys.length > 1) ? "are" : "is"
+      joined = self.class.configuration_keys.join(", ")
+      be = (self.class.configuration_keys.length > 1) ? "are" : "is"
       message = "#{self.class.class_key} is not configured correctly. #{joined} #{be} missing."
       raise ConfigurationError, message
     end

data/lib/mihari/analyzers/base.rb

@@ -79,6 +79,22 @@ module Mihari
         normalized_artifacts
       end
 
+      def result(...)
+        res = Try[StandardError] do
+          retry_on_error(
+            times: retry_times,
+            interval: retry_interval,
+            exponential_backoff: retry_exponential_backoff
+          ) do
+            call(...)
+          end
+        end
+
+        return res.recover { [] } if ignore_error?
+
+        res.to_result
+      end
+
       class << self
         #
         # Initialize an analyzer by query params

data/lib/mihari/analyzers/binaryedge.rb

@@ -24,8 +24,10 @@ module Mihari
         client.search_with_pagination(query, pagination_limit: pagination_limit).map(&:artifacts).flatten
       end
 
-      def configuration_keys
-        %w[binaryedge_api_key]
+      class << self
+        def configuration_keys
+          %w[binaryedge_api_key]
+        end
       end
 
       private

data/lib/mihari/analyzers/censys.rb

@@ -41,11 +41,13 @@ module Mihari
         configuration_keys? || (id? && secret?)
       end
 
-      #
-      # @return [Array<String>]
-      #
-      def configuration_keys
-        %w[censys_id censys_secret]
+      class << self
+        #
+        # @return [Array<String>]
+        #
+        def configuration_keys
+          %w[censys_id censys_secret]
+        end
       end
 
       private

data/lib/mihari/analyzers/circl.rb

@@ -6,7 +6,7 @@ module Mihari
     # CIRCL passive DNS/SSL analyzer
     #
     class CIRCL < Base
-      include Mixins::Refang
+      include Concerns::Refangable
 
       # @return [String, nil]
       attr_reader :type
@@ -47,8 +47,10 @@ module Mihari
         configuration_keys? || (username? && password?)
       end
 
-      def configuration_keys
-        %w[circl_passive_password circl_passive_username]
+      class << self
+        def configuration_keys
+          %w[circl_passive_password circl_passive_username]
+        end
       end
 
       private

data/lib/mihari/analyzers/crtsh.rb

@@ -9,20 +9,28 @@ module Mihari
       # @return [Boolean]
       attr_reader :exclude_expired
 
+      # @return [String, nil]
+      attr_reader :match
+
       #
       # @param [String] query
       # @param [Hash, nil] options
       # @param [Bool] exclude_expired
+      # @param [String, nil] match
       #
-      def initialize(query, options: nil, exclude_expired: true)
+      def initialize(query, options: nil, exclude_expired: true, match: nil)
         super(query, options: options)
 
         @exclude_expired = exclude_expired
+        @match = match
       end
 
       def artifacts
         exclude = exclude_expired ? "expired" : nil
-        client.search(query, exclude: exclude)
+        client.search(query, exclude: exclude, match: match).map do |result|
+          values = result["name_value"].to_s.lines.map(&:chomp).reject { |value| value.starts_with?("*.") }
+          values.map { |value| Models::Artifact.new(data: value, metadata: result) }
+        end.flatten
       end
 
       private

data/lib/mihari/analyzers/dnstwister.rb

@@ -6,7 +6,7 @@ module Mihari
     # DNSTwister analyzer
     #
     class DNSTwister < Base
-      include Mixins::Refang
+      include Concerns::Refangable
 
       # @return [String]
       attr_reader :type

data/lib/mihari/analyzers/feed.rb

@@ -1,8 +1,5 @@
 # frozen_string_literal: true
 
-require "mihari/feed/reader"
-require "mihari/feed/parser"
-
 module Mihari
   module Analyzers
     #
@@ -10,7 +7,7 @@ module Mihari
     #
     class Feed < Base
       # @return [Hash, nil]
-      attr_reader :data
+      attr_reader :form
 
       # @return [Hash, nil]
       attr_reader :json
@@ -37,37 +34,32 @@ module Mihari
       # @param [Hash, nil] headers
       # @param [Hash, nil] params
       # @param [Hash, nil] json
-      # @param [Hash, nil] data
+      # @param [form, nil] form
       # @param [String] selector
       #
-      def initialize(query, options: nil, method: "GET", headers: nil, params: nil, json: nil, data: nil, selector: "")
-        super(query, options: options)
+      # @param [Object] url
+      def initialize(url, options: nil, method: "GET", headers: nil, params: nil, json: nil, form: nil, selector: "")
+        super(url, options: options)
 
         @method = method
         @headers = headers || {}
         @params = params
         @json = json
-        @data = data
+        @form = form
         @selector = selector
       end
 
       def artifacts
-        Mihari::Feed::Parser.new(results).parse selector
+        data = Services::FeedReader.call(
+          url, headers: headers, method: method, params: params, json: json, form: form, timeout: timeout
+        )
+        Services::FeedParser.call(data, selector)
       end
 
       private
 
-      def results
-        reader = Mihari::Feed::Reader.new(
-          query,
-          method: method,
-          headers: headers,
-          timeout: timeout,
-          params: params,
-          json: json,
-          data: data
-        )
-        reader.read
+      def url
+        query
       end
     end
   end

data/lib/mihari/analyzers/fofa.rb

@@ -31,20 +31,18 @@ module Mihari
         end.flatten.compact
       end
 
-      def configuration_keys
-        %w[fofa_api_key fofa_email]
-      end
-
       def configured?
         api_key? && email?
       end
 
-      private
-
-      def api_key?
-        !api_key.nil?
+      class << self
+        def configuration_keys
+          %w[fofa_api_key fofa_email]
+        end
       end
 
+      private
+
       def email?
         !email.nil?
       end

data/lib/mihari/analyzers/greynoise.rb

@@ -27,8 +27,10 @@ module Mihari
         ).map(&:artifacts).flatten
       end
 
-      def configuration_keys
-        %w[greynoise_api_key]
+      class << self
+        def configuration_keys
+          %w[greynoise_api_key]
+        end
       end
 
       private

data/lib/mihari/analyzers/hunterhow.rb

@@ -44,8 +44,10 @@ module Mihari
         end.flatten
       end
 
-      def configuration_keys
-        %w[hunterhow_api_key]
+      class << self
+        def configuration_keys
+          %w[hunterhow_api_key]
+        end
       end
 
       private

data/lib/mihari/analyzers/onyphe.rb

@@ -29,8 +29,10 @@ module Mihari
         ).map(&:artifacts).flatten
       end
 
-      def configuration_keys
-        %w[onyphe_api_key]
+      class << self
+        def configuration_keys
+          %w[onyphe_api_key]
+        end
       end
 
       private

data/lib/mihari/analyzers/otx.rb

@@ -6,7 +6,7 @@ module Mihari
     # OTX analyzer
     #
     class OTX < Base
-      include Mixins::Refang
+      include Concerns::Refangable
 
       # @return [String, nil]
       attr_reader :type
@@ -38,8 +38,10 @@ module Mihari
         end
       end
 
-      def configuration_keys
-        %w[otx_api_key]
+      class << self
+        def configuration_keys
+          %w[otx_api_key]
+        end
       end
 
       private

data/lib/mihari/analyzers/passivetotal.rb

@@ -6,7 +6,7 @@ module Mihari
     # PassiveTotal analyzer
     #
     class PassiveTotal < Base
-      include Mixins::Refang
+      include Concerns::Refangable
 
       # @return [String, nil]
       attr_reader :type
@@ -35,11 +35,11 @@ module Mihari
       def artifacts
         case type
         when "domain", "ip"
-          client.passive_dns_search query
+          passive_dns_search
         when "mail"
-          client.reverse_whois_search query
+          reverse_whois_search
         when "hash"
-          client.ssl_search query
+          ssl_search
         else
           raise ValueError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
         end
@@ -49,11 +49,11 @@ module Mihari
         configuration_keys? || (username? && api_key?)
       end
 
-      def configuration_keys
-        %w[passivetotal_username passivetotal_api_key]
-      end
-
       class << self
+        def configuration_keys
+          %w[passivetotal_username passivetotal_api_key]
+        end
+
         #
         # @return [Array<String>, nil]
         #
@@ -64,6 +64,27 @@ module Mihari
 
       private
 
+      def passive_dns_search
+        res = client.passive_dns_search(query)
+        res["results"] || []
+      end
+
+      def reverse_whois_search
+        res = client.reverse_whois_search(query)
+        (res["results"] || []).map do |result|
+          data = result["domain"]
+          Models::Artifact.new(data: data, metadata: result)
+        end
+      end
+
+      def ssl_search
+        res = client.ssl_search(query)
+        (res["results"] || []).map do |result|
+          data = result["ipAddresses"]
+          data.map { |d| Models::Artifact.new(data: d, metadata: result) }
+        end.flatten
+      end
+
       def client
         Clients::PassiveTotal.new(username: username, api_key: api_key, timeout: timeout)
       end
@@ -80,10 +101,6 @@ module Mihari
       def username?
         !username.nil?
       end
-
-      def api_key?
-        !api_key.nil?
-      end
     end
   end
 end

data/lib/mihari/analyzers/pulsedive.rb

@@ -6,7 +6,7 @@ module Mihari
     # Pulsedive analyzer
     #
     class Pulsedive < Base
-      include Mixins::Refang
+      include Concerns::Refangable
 
       # @return [String, nil]
       attr_reader :type
@@ -43,8 +43,10 @@ module Mihari
         end
       end
 
-      def configuration_keys
-        %w[pulsedive_api_key]
+      class << self
+        def configuration_keys
+          %w[pulsedive_api_key]
+        end
       end
 
       private

data/lib/mihari/analyzers/securitytrails.rb

@@ -6,7 +6,7 @@ module Mihari
     # SecurityTrails
     #
     class SecurityTrails < Base
-      include Mixins::Refang
+      include Concerns::Refangable
 
       # @return [String, nil]
       attr_reader :type
@@ -33,21 +33,21 @@ module Mihari
       def artifacts
         case type
         when "domain"
-          client.domain_search query
+          domain_search
         when "ip"
-          client.ip_search query
+          ip_search
         when "mail"
-          client.mail_search query
+          mail_search
         else
           raise ValueError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
         end
       end
 
-      def configuration_keys
-        %w[securitytrails_api_key]
-      end
-
       class << self
+        def configuration_keys
+          %w[securitytrails_api_key]
+        end
+
         #
         # @return [Array<String>, nil]
         #
@@ -58,6 +58,30 @@ module Mihari
 
       private
 
+      def domain_search
+        client.get_all_dns_history(query, type: "a").map do |res|
+          (res["records"] || []).map do |record|
+            (record["values"] || []).map { |value| value["ip"] }
+          end.flatten.compact.uniq
+        end.flatten
+      end
+
+      def ip_search
+        res = client.ip_search(query)
+        (res["records"] || []).filter_map do |record|
+          data = record["hostname"]
+          Models::Artifact.new(data: data, metadata: record)
+        end
+      end
+
+      def mail_search
+        res = client.mail_search(query)
+        (res["records"] || []).filter_map do |record|
+          data = record["hostname"]
+          Models::Artifact.new(data: data, metadata: record)
+        end
+      end
+
       def client
         Clients::SecurityTrails.new(api_key: api_key, timeout: timeout)
       end

data/lib/mihari/analyzers/shodan.rb

@@ -27,8 +27,10 @@ module Mihari
         ).map(&:artifacts).flatten.uniq(&:data)
       end
 
-      def configuration_keys
-        %w[shodan_api_key]
+      class << self
+        def configuration_keys
+          %w[shodan_api_key]
+        end
       end
 
       private

data/lib/mihari/analyzers/urlscan.rb

@@ -37,8 +37,10 @@ module Mihari
         artifacts.select { |artifact| allowed_data_types.include? artifact.data_type }
       end
 
-      def configuration_keys
-        %w[urlscan_api_key]
+      class << self
+        def configuration_keys
+          %w[urlscan_api_key]
+        end
       end
 
       private