mihari 6.3.0 → 7.0.0

data/lib/mihari/analyzers/zoomeye.rb

@@ -40,8 +40,10 @@ module Mihari
         end
       end
 
-      def configuration_keys
-        %w[zoomeye_api_key]
+      class << self
+        def configuration_keys
+          %w[zoomeye_api_key]
+        end
       end
 
       private

data/lib/mihari/cli/{main.rb → application.rb}

@@ -4,11 +4,10 @@ require "thor"
 require "thor/hollaback"
 
 # Commands
-require "mihari/commands/mixins"
-
 require "mihari/commands/alert"
 require "mihari/commands/database"
 require "mihari/commands/search"
+require "mihari/commands/sidekiq"
 require "mihari/commands/version"
 require "mihari/commands/web"
 
@@ -16,23 +15,27 @@ require "mihari/commands/web"
 require "mihari/cli/base"
 
 require "mihari/cli/alert"
+require "mihari/cli/artifact"
+require "mihari/cli/config"
 require "mihari/cli/database"
 require "mihari/cli/rule"
+require "mihari/cli/tag"
 
 module Mihari
   module CLI
     #
     # Main CLI class
     #
-    class Main < Base
-      class_option :debug, desc: "Sets up debug mode", aliases: ["-d"], type: :boolean
+    class App < Base
+      class_option :debug, desc: "Set up debug mode", aliases: ["-d"], type: :boolean
       class_around :safe_execute
 
       include Mihari::Commands::Search
+      include Mihari::Commands::Sidekiq
       include Mihari::Commands::Version
       include Mihari::Commands::Web
 
-      include Mihari::Mixins::UnwrapError
+      include Concerns::ErrorUnwrappable
 
       no_commands do
         def safe_execute
@@ -62,6 +65,15 @@ module Mihari
 
       desc "alert", "Sub commands for alert"
       subcommand "alert", Alert
+
+      desc "artifact", "Sub commands for artifact"
+      subcommand "artifact", Artifact
+
+      desc "tag", "Sub commands for tag"
+      subcommand "tag", Tag
+
+      desc "config", "Sub commands for config"
+      subcommand "config", Config
     end
   end
 end

data/lib/mihari/cli/artifact.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require "mihari/commands/artifact"
+
+module Mihari
+  module CLI
+    #
+    # Artifact CLI class (mihari artifact ...)
+    #
+    class Artifact < Base
+      include Mihari::Commands::Artifact
+    end
+  end
+end

data/lib/mihari/cli/config.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require "mihari/commands/config"
+
+module Mihari
+  module CLI
+    #
+    # Config CLI class (mihari config ...)
+    #
+    class Config < Base
+      include Mihari::Commands::Config
+    end
+  end
+end

data/lib/mihari/cli/rule.rb

@@ -9,6 +9,7 @@ module Mihari
     #
     class Rule < Base
       include Mihari::Commands::Rule
+      include Mihari::Commands::Search
     end
   end
 end

data/lib/mihari/cli/tag.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require "mihari/commands/tag"
+
+module Mihari
+  module CLI
+    #
+    # Tag CLI class (mihari tag ...)
+    #
+    class Tag < Base
+      include Mihari::Commands::Tag
+    end
+  end
+end

data/lib/mihari/clients/base.rb

@@ -47,10 +47,10 @@ module Mihari
       #
       # @param [String] path
       #
-      # @return [String]
+      # @return [URI]
       #
       def url_for(path)
-        base_url + path
+        URI.join base_url, path
       end
 
       #

data/lib/mihari/clients/binaryedge.rb

@@ -14,7 +14,7 @@ module Mihari
       # @param [Integer] pagination_interval
       #
       def initialize(
-        base_url = "https://api.binaryedge.io/v2",
+        base_url = "https://api.binaryedge.io",
         api_key:,
         headers: {},
         pagination_interval: Mihari.config.pagination_interval,
@@ -38,7 +38,7 @@ module Mihari
           page: page,
           only_ips: only_ips
         }.compact
-        Structs::BinaryEdge::Response.from_dynamic! get_json("/query/search", params: params)
+        Structs::BinaryEdge::Response.from_dynamic! get_json("/v2/query/search", params: params)
       end
 
       #

data/lib/mihari/clients/crtsh.rb

@@ -22,17 +22,10 @@ module Mihari
       # @param [String, nil] match "=", "ILIKE", "LIKE", "single", "any" or nil
       # @param [String, nil] exclude "expired" or nil
       #
-      # @return [Array<Mihari::Models::Artifact>]
+      # @return [Array<Hash>]
       #
       def search(identity, match: nil, exclude: nil)
-        params = { identity: identity, match: match, exclude: exclude, output: "json" }.compact
-
-        # @type [Array[Hash]]
-        parsed = get_json("/", params: params)
-        parsed.map do |result|
-          values = result["name_value"].to_s.lines.map(&:chomp)
-          values.map { |value| Models::Artifact.new(data: value, metadata: result) }
-        end.flatten
+        get_json("/", params: { identity: identity, match: match, exclude: exclude, output: "json" }.compact)
       end
     end
   end

data/lib/mihari/clients/fofa.rb

@@ -26,7 +26,7 @@ module Mihari
       #
       # @param [Object] email
       def initialize(
-        base_url = "https://fofa.info/",
+        base_url = "https://fofa.info",
         api_key:,
         email:,
         headers: {},

data/lib/mihari/clients/hunterhow.rb

@@ -21,7 +21,7 @@ module Mihari
       # @param [Integer, nil] timeout
       #
       def initialize(
-        base_url = "https://api.hunter.how/",
+        base_url = "https://api.hunter.how",
         api_key:,
         headers: {},
         pagination_interval: Mihari.config.pagination_interval,

data/lib/mihari/clients/mmdb.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Clients
+    #
+    # MMDB API client
+    #
+    class MMDB < Base
+      #
+      # @param [String] base_url
+      # @param [Hash] headers
+      # @param [Integer, nil] timeout
+      #
+      def initialize(base_url = "https://ip.circl.lu", headers: {}, timeout: nil)
+        super(base_url, headers: headers, timeout: timeout)
+      end
+
+      #
+      # @param [String] ip
+      #
+      # @return [Mihari::Structs::MMDB::Response]
+      #
+      def query(ip)
+        Structs::MMDB::Response.from_dynamic! get_json("/geolookup/#{ip}")
+      end
+    end
+  end
+end

data/lib/mihari/clients/passivetotal.rb

@@ -29,12 +29,11 @@ module Mihari
       #
       # @param [String] query
       #
-      # @return [Array<String>]
+      # @return [Hash]
       #
       def passive_dns_search(query)
         params = { query: query }
-        res = get_json("/v2/dns/passive/unique", params: params)
-        res["results"] || []
+        get_json("/v2/dns/passive/unique", params: params)
       end
 
       #
@@ -42,19 +41,13 @@ module Mihari
       #
       # @param [String] query
       #
-      # @return [Array<Mihari::Models::Artifact>]
+      # @return [Hash]
       #
       def reverse_whois_search(query)
-        params = {
+        get_json("/v2/whois/search", params: {
           query: query,
           field: "email"
-        }.compact
-        res = get_json("/v2/whois/search", params: params)
-        results = res["results"] || []
-        results.map do |result|
-          data = result["domain"]
-          Models::Artifact.new(data: data, metadata: result)
-        end.flatten
+        }.compact)
       end
 
       #
@@ -62,16 +55,10 @@ module Mihari
       #
       # @param [String] query
       #
-      # @return [Array<Mihari::Models::Artifact>]
+      # @return [Hash]
       #
       def ssl_search(query)
-        params = { query: query }
-        res = get_json("/v2/ssl-certificate/history", params: params)
-        results = res["results"] || []
-        results.map do |result|
-          data = result["ipAddresses"]
-          data.map { |d| Models::Artifact.new(data: d, metadata: result) }
-        end.flatten
+        get_json("/v2/ssl-certificate/history", params: { query: query })
       end
     end
   end

data/lib/mihari/clients/securitytrails.rb

@@ -20,33 +20,15 @@ module Mihari
         super(base_url, headers: headers, timeout: timeout)
       end
 
-      #
-      # Domain search
-      #
-      # @param [String] query
-      #
-      # @return [Array<String>]
-      #
-      def domain_search(query)
-        records = get_all_dns_history(query, type: "a")
-        records.map do |record|
-          (record["values"] || []).map { |value| value["ip"] }
-        end.flatten.compact.uniq
-      end
-
       #
       # IP search
       #
       # @param [String] query
       #
-      # @return [Array<Mihari::Models::Artifact>]
+      # @return [Hash]
       #
       def ip_search(query)
-        records = search_by_ip(query)
-        records.filter_map do |record|
-          data = record["hostname"]
-          Models::Artifact.new(data: data, metadata: record)
-        end
+        search_by_ip(query)
       end
 
       #
@@ -54,54 +36,48 @@ module Mihari
       #
       # @param [String] query
       #
-      # @return [Array<String>]
+      # @return [Hash]
       #
       def mail_search(query)
-        records = search_by_mail(query)
-        records.filter_map do |record|
-          data = record["hostname"]
-          Models::Artifact.new(data: data, metadata: record)
-        end
+        search_by_mail(query)
       end
 
       #
       # @param [String] mail
       #
-      # @return [Array<Hash>]
+      # @return [Hash]
       #
       def search_by_mail(mail)
-        res = post_json "/v1/domains/list", json: { filter: { whois_email: mail } }
-        res["records"] || []
+        post_json "/v1/domains/list", json: { filter: { whois_email: mail } }
       end
 
       #
       # @param [String] ip
       #
-      # @return [Array<Hash>]
+      # @return [Hash]
       #
       def search_by_ip(ip)
-        res = post_json "/v1/domains/list", json: { filter: { ipv4: ip } }
-        res["records"] || []
+        post_json "/v1/domains/list", json: { filter: { ipv4: ip } }
       end
 
       #
       # @param [String] domain
       # @param [String] type
+      # @param [Integer] page
       #
-      # @return [Array<Hash>]
+      # @return [Enumerable<Hash>]
       #
-      def get_all_dns_history(domain, type:)
-        first_page = get_dns_history(domain, type: type, page: 1)
+      def get_all_dns_history(domain, type:, page: 1)
+        Enumerator.new do |y|
+          res = get_dns_history(domain, type: type, page: page)
+          y.yield res
 
-        pages = first_page["pages"].to_i
-        records = first_page["records"] || []
+          pages = res["pages"].to_i
 
-        (2..pages).each do |page_idx|
-          next_page = get_dns_history(domain, type: type, page: page_idx)
-          records << next_page["records"]
+          (page + 1..pages).each do |page|
+            y.yield get_dns_history(domain, type: type, page: page)
+          end
         end
-
-        records.flatten
       end
 
       private
@@ -111,7 +87,7 @@ module Mihari
       # @param [String] type
       # @param [Integer] page
       #
-      # @return [Array<Hash>]
+      # @return [Hash]
       #
       def get_dns_history(domain, type:, page:)
         get_json "/v1/history/#{domain}/dns/#{type}", params: { page: page }

data/lib/mihari/clients/shodan_internet_db.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Clients
+    #
+    # Shodan Internet DB API client
+    #
+    class ShodanInternetDB < Base
+      #
+      # @param [String] base_url
+      # @param [Hash] headers
+      # @param [Integer, nil] timeout
+      #
+      def initialize(base_url = "https://internetdb.shodan.io", headers: {}, timeout: nil)
+        super(base_url, headers: headers, timeout: timeout)
+      end
+
+      #
+      # @param [String] ip
+      #
+      # @return [Mihari::Structs::Shodan::InternetDBResponse]
+      #
+      def query(ip)
+        Structs::Shodan::InternetDBResponse.from_dynamic! get_json("/#{ip}")
+      end
+    end
+  end
+end

data/lib/mihari/clients/the_hive.rb

@@ -6,20 +6,22 @@ module Mihari
     # TheHive API client
     #
     class TheHive < Base
+      attr_reader :api_version
+
       #
       # @param [String] base_url
       # @param [String, nil] api_key
-      # @param [String, nil] api_version
+      # @param [String] api_version
       # @param [Hash] headers
       # @param [Integer, nil] timeout
       #
-      def initialize(base_url, api_key:, api_version:, headers: {}, timeout: nil)
+      def initialize(base_url, api_key:, api_version: "v1", headers: {}, timeout: nil)
         raise(ArgumentError, "api_key is required") unless api_key
 
-        base_url += "/#{api_version}" unless api_version.nil?
         headers["authorization"] = "Bearer #{api_key}"
-
         super(base_url, headers: headers, timeout: timeout)
+
+        @api_version = api_version
       end
 
       #
@@ -29,7 +31,7 @@ module Mihari
       #
       def alert(json)
         json = json.to_camelback_keys.compact
-        post_json("/alert", json: json)
+        post_json("/api/#{api_version}/alert", json: json)
       end
     end
   end

data/lib/mihari/commands/alert.rb

@@ -7,30 +7,72 @@ module Mihari
     #
     module Alert
       class << self
+        # rubocop:disable Metrics/AbcSize
         def included(thor)
           thor.class_eval do
-            include Dry::Monads[:result, :try]
-            include Mixins
+            include Concerns::DatabaseConnectable
 
-            desc "add [PATH]", "Add an alert"
+            desc "create [PATH]", "Create an alert"
             around :with_db_connection
             #
             # @param [String] path
             #
-            def add(path)
-              result = Dry::Monads::Try[StandardError] do
-                # @type [Mihari::Services::AlertProxy]
-                proxy = Mihari::Services::AlertBuilder.call(path)
-                Mihari::Services::AlertRunner.call proxy
-              end.to_result
-
+            def create(path)
               # @type [Mihari::Models::Alert]
-              alert = result.value!
+              alert = Dry::Monads::Try[StandardError] do
+                raise ArgumentError, "#{path} not found" unless Pathname(path).exist?
+
+                params = YAML.safe_load(
+                  ERB.new(File.read(path)).result,
+                  permitted_classes: [Date, Symbol]
+                )
+                Services::AlertCreator.call params
+              end.value!
               data = Entities::Alert.represent(alert)
               puts JSON.pretty_generate(data.as_json)
             end
+
+            desc "list [QUERY]", "List/search alerts"
+            around :with_db_connection
+            method_option :page, type: :numeric, default: 1
+            method_option :limit, type: :numeric, default: 10
+            #
+            # @param [String] q
+            #
+            def list(q = "")
+              filter = Structs::Filters::Search.new(q: q, page: options["page"], limit: options["limit"])
+              value = Services::AlertSearcher.result(filter).value!
+              data = Entities::AlertsWithPagination.represent(
+                results: value.results,
+                total: value.total,
+                current_page: value.filter[:page].to_i,
+                page_size: value.filter[:limit].to_i
+              )
+              puts JSON.pretty_generate(data.as_json)
+            end
+
+            desc "get [ID]", "Get an alert"
+            around :with_db_connection
+            #
+            # @param [Integer] id
+            #
+            def get(id)
+              value = Services::AlertGetter.result(id).value!
+              data = Entities::Alert.represent(value)
+              puts JSON.pretty_generate(data.as_json)
+            end
+
+            desc "delete [ID]", "Delete an alert"
+            around :with_db_connection
+            #
+            # @param [Integer] id
+            #
+            def delete(id)
+              Services::AlertDestroyer.result(id).value!
+            end
           end
         end
+        # rubocop:enable Metrics/AbcSize
       end
     end
   end

data/lib/mihari/commands/artifact.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Commands
+    #
+    # Artifact sub-commands
+    #
+    module Artifact
+      class << self
+        def included(thor)
+          thor.class_eval do
+            include Concerns::DatabaseConnectable
+
+            desc "list [QUERY]", "List/search artifacts"
+            around :with_db_connection
+            method_option :page, type: :numeric, default: 1
+            method_option :limit, type: :numeric, default: 10
+            #
+            # @param [String] q
+            #
+            def list(q = "")
+              filter = Structs::Filters::Search.new(q: q, page: options["page"], limit: options["limit"])
+              value = Services::ArtifactSearcher.result(filter).value!
+              data = Entities::ArtifactsWithPagination.represent(
+                results: value.results,
+                total: value.total,
+                current_page: value.filter[:page].to_i,
+                page_size: value.filter[:limit].to_i
+              )
+              puts JSON.pretty_generate(data.as_json)
+            end
+
+            desc "get [ID]", "Get an artifact"
+            around :with_db_connection
+            #
+            # @param [Integer] id
+            #
+            def get(id)
+              value = Services::ArtifactGetter.result(id).value!
+              data = Entities::Artifact.represent(value)
+              puts JSON.pretty_generate(data.as_json)
+            end
+
+            desc "enrich [ID]", "Enrich an artifact"
+            around :with_db_connection
+            #
+            # @param [Integer] id
+            #
+            def enrich(id)
+              Services::ArtifactEnricher.result(id).value!
+            end
+
+            desc "delete [ID]", "Delete an artifact"
+            around :with_db_connection
+            #
+            # @param [Integer] id
+            #
+            def delete(id)
+              Services::ArtifactDestroyer.result(id).value!
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mihari/commands/config.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Commands
+    #
+    # Config sub-commands
+    #
+    module Config
+      class << self
+        def included(thor)
+          thor.class_eval do
+            desc "list", "List config"
+            def list
+              configs = Services::ConfigSearcher.call
+              data = configs.map { |config| Entities::Config.represent(config) }
+              puts JSON.pretty_generate(data.as_json)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mihari/commands/database.rb

@@ -9,7 +9,7 @@ module Mihari
       class << self
         def included(thor)
           thor.class_eval do
-            include Mixins
+            include Concerns::DatabaseConnectable
 
             desc "migrate", "Migrate DB schemas"
             around :with_db_connection