mihari 5.7.0 → 5.7.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f226f30917400448aecfc83808609ff3eb180228c2d10d797d3409cc3ecdb9c0
-  data.tar.gz: 5ce20637fed160cdecb7d8c22726b44670834843f2e77f5ef235f943a6321701
+  metadata.gz: 5c7fa4de39ec815c08930311d4ed642857236021301508c47d298cba157dd811
+  data.tar.gz: 1598a9992be6739957bc5145c11d93f8a869588d42985799d527703178f78a8f
 SHA512:
-  metadata.gz: 3900c2cc6a8ac2010b4579f574be0ce4d620a180dcfadf00c97b7597a7bccd79de9efdfdce402b40f06d7afeaaec0062210796391123e3a145f54d3be4270261
-  data.tar.gz: 7655638dd621bf03f0d9178eb2e0ca6bf4d161b6563c266380d3cc8bf37bef692bc06765ef3050aaf7683a0f1be2ca51d8a9d7e7c3f34a9e184389fcd31b2885
+  metadata.gz: 6c96715d51778d724b4df1c502d3083c4af084997019cc7ef43fc5d131cfeb02c4b0f06d6353c0d8e75e26dec0fb20038fa36718f9a65bb1a3a771224442bc3e
+  data.tar.gz: 655c0efa77ff2602fef509b351ebf02ab9c222728ddfcfc0da36ba8fa27a4e1518f1d70406b5ba3ace86012c196954a98c9af10a4de58c5dcdf73bbc760d2056

data/.rubocop.yml

@@ -3,7 +3,7 @@ Style/HashSyntax:
 Style/StringLiterals:
   EnforcedStyle: double_quotes
 Metrics/BlockLength:
-  Max: 100
+  Max: 150
   Exclude:
     - "spec/**/*"
     - "*.gemspec"

data/lib/mihari/actor.rb

@@ -2,9 +2,11 @@
 
 module Mihari
   #
-  # Base class for Analyzer, Emitter and Enricher
+  # Yet another base service class for Analyzer, Emitter and Enricher
   #
-  class Actor < Service
+  class Actor
+    include Dry::Monads[:result, :try]
+
     include Mixins::Configurable
     include Mixins::Retriable
 
@@ -57,13 +59,17 @@ module Mihari
       raise ConfigurationError, message
     end
 
-    def result
+    def call(*args, **kwargs)
+      raise NotImplementedError, "You must implement #{self.class}##{__method__}"
+    end
+
+    def result(...)
       Try[StandardError] do
         retry_on_error(
           times: retry_times,
           interval: retry_interval,
           exponential_backoff: retry_exponential_backoff
-        ) { call }
+        ) { call(...) }
       end.to_result
     end
 

data/lib/mihari/cli/main.rb

@@ -4,6 +4,8 @@ require "thor"
 require "thor/hollaback"
 
 # Commands
+require "mihari/commands/mixins"
+
 require "mihari/commands/alert"
 require "mihari/commands/database"
 require "mihari/commands/search"

data/lib/mihari/clients/base.rb

@@ -41,7 +41,7 @@ module Mihari
       # @return [::HTTP::Client]
       #
       def http
-        HTTP::Factory.build headers: headers, timeout: timeout
+        @http ||= HTTP::Factory.build(headers: headers, timeout: timeout)
       end
 
       #
@@ -63,6 +63,17 @@ module Mihari
         http.get(url_for(path), params: params)
       end
 
+      #
+      # @param [String] path
+      # @param [Hash, nil] params
+      #
+      # @return [Hash]
+      #
+      def get_json(path, params: nil)
+        res = get(path, params: params)
+        JSON.parse res.body.to_s
+      end
+
       #
       # @param [String] path
       # @param [Hash, nil] json
@@ -72,6 +83,17 @@ module Mihari
       def post(path, json: {})
         http.post(url_for(path), json: json)
       end
+
+      #
+      # @param [String] path
+      # @param [Hash, nil] json
+      #
+      # @return [Hash]
+      #
+      def post_json(path, json: {})
+        res = http.post(url_for(path), json: json)
+        JSON.parse res.body.to_s
+      end
     end
   end
 end

data/lib/mihari/clients/binaryedge.rb

@@ -38,9 +38,7 @@ module Mihari
           page: page,
           only_ips: only_ips
         }.compact
-
-        res = get("/query/search", params: params)
-        Structs::BinaryEdge::Response.from_dynamic! JSON.parse(res.body.to_s)
+        Structs::BinaryEdge::Response.from_dynamic! get_json("/query/search", params: params)
       end
 
       #

data/lib/mihari/clients/censys.rb

@@ -46,8 +46,7 @@ module Mihari
       #
       def search(query, per_page: nil, cursor: nil)
         params = { q: query, per_page: per_page, cursor: cursor }.compact
-        res = get("/api/v2/hosts/search", params: params)
-        Structs::Censys::Response.from_dynamic! JSON.parse(res.body.to_s)
+        Structs::Censys::Response.from_dynamic! get_json("/api/v2/hosts/search", params: params)
       end
 
       #

data/lib/mihari/clients/crtsh.rb

@@ -27,9 +27,8 @@ module Mihari
       def search(identity, match: nil, exclude: nil)
         params = { identity: identity, match: match, exclude: exclude, output: "json" }.compact
 
-        res = get("/", params: params)
-        parsed = JSON.parse(res.body.to_s)
-
+        # @type [Array[Hash]]
+        parsed = get_json("/", params: params)
         parsed.map do |result|
           values = result["name_value"].to_s.lines.map(&:chomp)
           values.map { |value| Models::Artifact.new(data: value, metadata: result) }

data/lib/mihari/clients/dnstwister.rb

@@ -23,8 +23,7 @@ module Mihari
       # @return [Array<String>]
       #
       def fuzz(domain)
-        res = get("/api/fuzz/#{to_hex(domain)}")
-        res = JSON.parse(res.body.to_s)
+        res = get_json("/api/fuzz/#{to_hex(domain)}")
         fuzzy_domains = res["fuzzy_domains"] || []
         fuzzy_domains.map { |d| d["domain"] }
       end

data/lib/mihari/clients/fofa.rb

@@ -52,8 +52,7 @@ module Mihari
       def search(query, page:, size: PAGE_SIZE)
         qbase64 = Base64.urlsafe_encode64(query)
         params = { qbase64: qbase64, size: size, page: page, email: email, key: api_key }.compact
-        res = get("/api/v1/search/all", params: params)
-        Structs::Fofa::Response.from_dynamic! JSON.parse(res.body.to_s)
+        Structs::Fofa::Response.from_dynamic! get_json("/api/v1/search/all", params: params)
       end
 
       #
@@ -71,7 +70,6 @@ module Mihari
             y.yield res
 
             break if res.error
-
             break if (res.results || []).length < size
 
             sleep_pagination_interval

data/lib/mihari/clients/greynoise.rb

@@ -39,8 +39,7 @@ module Mihari
       #
       def gnql_search(query, size: PAGE_SIZE, scroll: nil)
         params = { query: query, size: size, scroll: scroll }.compact
-        res = get("/v2/experimental/gnql", params: params)
-        Structs::GreyNoise::Response.from_dynamic! JSON.parse(res.body.to_s)
+        Structs::GreyNoise::Response.from_dynamic! get_json("/v2/experimental/gnql", params: params)
       end
 
       #

data/lib/mihari/clients/hunterhow.rb

@@ -52,8 +52,7 @@ module Mihari
           end_time: end_time,
           "api-key": api_key
         }.compact
-        res = get("/search", params: params)
-        Structs::HunterHow::Response.from_dynamic! JSON.parse(res.body.to_s)
+        Structs::HunterHow::Response.from_dynamic! get_json("/search", params: params)
       end
 
       #

data/lib/mihari/clients/misp.rb

@@ -27,8 +27,7 @@ module Mihari
       # @return [Hash]
       #
       def create_event(payload)
-        res = post("/events/add", json: payload)
-        JSON.parse(res.body.to_s)
+        post_json("/events/add", json: payload)
       end
     end
   end

data/lib/mihari/clients/onyphe.rb

@@ -40,8 +40,7 @@ module Mihari
       #
       def datascan(query, page: 1)
         params = { page: page, apikey: api_key }
-        res = get("/api/v2/simple/datascan/#{query}", params: params)
-        Structs::Onyphe::Response.from_dynamic! JSON.parse(res.body.to_s)
+        Structs::Onyphe::Response.from_dynamic! get_json("/api/v2/simple/datascan/#{query}", params: params)
       end
 
       #

data/lib/mihari/clients/otx.rb

@@ -65,7 +65,7 @@ module Mihari
       # @return [Hash]
       #
       def query_by_ip(ip)
-        _get "/api/v1/indicators/IPv4/#{ip}/passive_dns"
+        get_json "/api/v1/indicators/IPv4/#{ip}/passive_dns"
       end
 
       #
@@ -74,19 +74,7 @@ module Mihari
       # @return [Hash]
       #
       def query_by_domain(domain)
-        _get "/api/v1/indicators/domain/#{domain}/passive_dns"
-      end
-
-      private
-
-      #
-      # @param [String] path
-      #
-      # @return [Hash]
-      #
-      def _get(path)
-        res = get(path)
-        JSON.parse(res.body.to_s)
+        get_json "/api/v1/indicators/domain/#{domain}/passive_dns"
       end
     end
   end

data/lib/mihari/clients/passivetotal.rb

@@ -33,7 +33,7 @@ module Mihari
       #
       def passive_dns_search(query)
         params = { query: query }
-        res = _get("/v2/dns/passive/unique", params: params)
+        res = get_json("/v2/dns/passive/unique", params: params)
         res["results"] || []
       end
 
@@ -49,7 +49,7 @@ module Mihari
           query: query,
           field: "email"
         }.compact
-        res = _get("/v2/whois/search", params: params)
+        res = get_json("/v2/whois/search", params: params)
         results = res["results"] || []
         results.map do |result|
           data = result["domain"]
@@ -66,26 +66,13 @@ module Mihari
       #
       def ssl_search(query)
         params = { query: query }
-        res = _get("/v2/ssl-certificate/history", params: params)
+        res = get_json("/v2/ssl-certificate/history", params: params)
         results = res["results"] || []
         results.map do |result|
           data = result["ipAddresses"]
           data.map { |d| Models::Artifact.new(data: d, metadata: result) }
         end.flatten
       end
-
-      private
-
-      #
-      # @param [String] path
-      # @param [Hash] params
-      #
-      # @return [Hash]
-      #
-      def _get(path, params: {})
-        res = get(path, params: params)
-        JSON.parse(res.body.to_s)
-      end
     end
   end
 end

data/lib/mihari/clients/publsedive.rb

@@ -29,7 +29,7 @@ module Mihari
       # @return [Hash]
       #
       def get_indicator(ip_or_domain)
-        _get "/api/info.php", params: { indicator: ip_or_domain }
+        get_json "/api/info.php", params: { indicator: ip_or_domain, key: api_key }
       end
 
       #
@@ -38,22 +38,7 @@ module Mihari
       # @return [Hash]
       #
       def get_properties(indicator_id)
-        _get "/api/info.php", params: { iid: indicator_id, get: "properties" }
-      end
-
-      private
-
-      #
-      # @param [String] path
-      # @param [Hash] params
-      #
-      # @return [Hash]
-      #
-      def _get(path, params: {})
-        params["key"] = api_key
-
-        res = get(path, params: params)
-        JSON.parse res.body.to_s
+        get_json "/api/info.php", params: { iid: indicator_id, get: "properties", key: api_key }
       end
     end
   end

data/lib/mihari/clients/securitytrails.rb

@@ -70,7 +70,7 @@ module Mihari
       # @return [Array<Hash>]
       #
       def search_by_mail(mail)
-        res = _post "/v1/domains/list", json: { filter: { whois_email: mail } }
+        res = post_json "/v1/domains/list", json: { filter: { whois_email: mail } }
         res["records"] || []
       end
 
@@ -80,7 +80,7 @@ module Mihari
       # @return [Array<Hash>]
       #
       def search_by_ip(ip)
-        res = _post "/v1/domains/list", json: { filter: { ipv4: ip } }
+        res = post_json "/v1/domains/list", json: { filter: { ipv4: ip } }
         res["records"] || []
       end
 
@@ -114,29 +114,7 @@ module Mihari
       # @return [Array<Hash>]
       #
       def get_dns_history(domain, type:, page:)
-        _get "/v1/history/#{domain}/dns/#{type}", params: { page: page }
-      end
-
-      #
-      # @param [String] path
-      # @param [Hash, nil] params
-      #
-      # @return [Hash]
-      #
-      def _get(path, params:)
-        res = get(path, params: params)
-        JSON.parse(res.body.to_s)
-      end
-
-      #
-      # @param [String] path
-      # @param [Hash, nil] json
-      #
-      # @return [Hash]
-      #
-      def _post(path, json:)
-        res = post(path, json: json)
-        JSON.parse(res.body.to_s)
+        get_json "/v1/history/#{domain}/dns/#{type}", params: { page: page }
       end
     end
   end

data/lib/mihari/clients/shodan.rb

@@ -46,8 +46,7 @@ module Mihari
           minify: minify,
           key: api_key
         }
-        res = get("/shodan/host/search", params: params)
-        Structs::Shodan::Response.from_dynamic! JSON.parse(res.body.to_s)
+        Structs::Shodan::Response.from_dynamic! get_json("/shodan/host/search", params: params)
       end
 
       #

data/lib/mihari/clients/the_hive.rb

@@ -29,8 +29,7 @@ module Mihari
       #
       def alert(json)
         json = json.to_camelback_keys.compact
-        res = post("/alert", json: json)
-        JSON.parse(res.body.to_s)
+        post_json("/alert", json: json)
       end
     end
   end

data/lib/mihari/clients/urlscan.rb

@@ -36,8 +36,7 @@ module Mihari
       #
       def search(q, size: nil, search_after: nil)
         params = { q: q, size: size, search_after: search_after }.compact
-        res = get("/api/v1/search/", params: params)
-        Structs::Urlscan::Response.from_dynamic! JSON.parse(res.body.to_s)
+        Structs::Urlscan::Response.from_dynamic! get_json("/api/v1/search/", params: params)
       end
 
       #

data/lib/mihari/clients/virustotal.rb

@@ -33,7 +33,7 @@ module Mihari
       # @return [Hash]
       #
       def domain_search(query)
-        _get("/api/v3/domains/#{query}/resolutions")
+        get_json "/api/v3/domains/#{query}/resolutions"
       end
 
       #
@@ -42,7 +42,7 @@ module Mihari
       # @return [Hash]
       #
       def ip_search(query)
-        _get("/api/v3/ip_addresses/#{query}/resolutions")
+        get_json "/api/v3/ip_addresses/#{query}/resolutions"
       end
 
       #
@@ -53,8 +53,7 @@ module Mihari
       #
       def intel_search(query, cursor: nil)
         params = { query: query, cursor: cursor }.compact
-        res = _get("/api/v3/intelligence/search", params: params)
-        Structs::VirusTotalIntelligence::Response.from_dynamic! res
+        Structs::VirusTotalIntelligence::Response.from_dynamic! get_json("/api/v3/intelligence/search", params: params)
       end
 
       #
@@ -79,19 +78,6 @@ module Mihari
           end
         end
       end
-
-      private
-
-      #
-      # @param [String] path
-      # @param [Hash] params
-      #
-      # @return [Hash]
-      #
-      def _get(path, params: {})
-        res = get(path, params: params)
-        JSON.parse(res.body.to_s)
-      end
     end
   end
 end

data/lib/mihari/clients/zoomeye.rb

@@ -30,6 +30,13 @@ module Mihari
         super(base_url, headers: headers, pagination_interval: pagination_interval, timeout: timeout)
       end
 
+      #
+      # @return [::HTTP::Client]
+      #
+      def http
+        @http ||= HTTP::Factory.build(headers: headers, timeout: timeout, raise_exception: false)
+      end
+
       #
       # Search the Host devices
       #
@@ -45,8 +52,7 @@ module Mihari
           page: page,
           facets: facets
         }.compact
-
-        _get("/host/search", params: params)
+        get_json "/host/search", params: params
       end
 
       #
@@ -88,8 +94,7 @@ module Mihari
           page: page,
           facets: facets
         }.compact
-
-        _get("/web/search", params: params)
+        get_json "/web/search", params: params
       end
 
       #
@@ -115,21 +120,6 @@ module Mihari
           end
         end
       end
-
-      private
-
-      #
-      # @param [String] path
-      # @param [Hash] params
-      #
-      # @return [Hash, nil]
-      #
-      def _get(path, params: {})
-        res = get(path, params: params)
-        JSON.parse(res.body.to_s)
-      rescue HTTPError
-        nil
-      end
     end
   end
 end

data/lib/mihari/commands/alert.rb

@@ -10,22 +10,24 @@ module Mihari
         def included(thor)
           thor.class_eval do
             include Dry::Monads[:result, :try]
+            include Mixins
 
             desc "add [PATH]", "Add an alert"
+            around :with_db_connection
             #
             # @param [String] path
             #
             def add(path)
-              Mihari::Database.with_db_connection do
-                builder = Services::AlertBuilder.new(path)
+              result = Dry::Monads::Try[StandardError] do
+                # @type [Mihari::Services::AlertProxy]
+                proxy = Mihari::Services::AlertBuilder.call(path)
+                Mihari::Services::AlertRunner.call proxy
+              end.to_result
 
-                runner_result_l = ->(proxy) { Services::AlertRunner.new(proxy).result }
-                result = builder.result.bind(runner_result_l)
-
-                alert = result.value!
-                data = Entities::Alert.represent(alert)
-                puts JSON.pretty_generate(data.as_json)
-              end
+              # @type [Mihari::Models::Alert]
+              alert = result.value!
+              data = Entities::Alert.represent(alert)
+              puts JSON.pretty_generate(data.as_json)
             end
           end
         end

data/lib/mihari/commands/database.rb

@@ -9,7 +9,10 @@ module Mihari
       class << self
         def included(thor)
           thor.class_eval do
+            include Mixins
+
             desc "migrate", "Migrate DB schemas"
+            around :with_db_connection
             method_option :verbose, type: :boolean, default: true
             #
             # @param [String] direction
@@ -17,7 +20,7 @@ module Mihari
             def migrate(direction = "up")
               ActiveRecord::Migration.verbose = options["verbose"]
 
-              Mihari::Database.with_db_connection { Mihari::Database.migrate direction.to_sym }
+              Mihari::Database.migrate direction.to_sym
             end
           end
         end

data/lib/mihari/commands/mixins.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Commands
+    module Mixins
+      def with_db_connection(&block)
+        Mihari::Database.with_db_connection(&block)
+      end
+    end
+  end
+end

data/lib/mihari/commands/search.rb

@@ -10,8 +10,10 @@ module Mihari
         def included(thor)
           thor.class_eval do
             include Dry::Monads[:try, :result]
+            include Mixins
 
             desc "search [PATH_OR_ID]", "Search by a rule (Outputs null if there is no new finding)"
+            around :with_db_connection
             method_option :force_overwrite, type: :boolean, aliases: "-f", desc: "Force overwriting a rule"
             #
             # Search by a rule
@@ -19,43 +21,22 @@ module Mihari
             # @param [String] path_or_id
             #
             def search(path_or_id)
-              Mihari::Database.with_db_connection do
-                builder = Services::RuleBuilder.new(path_or_id)
+              result = Dry::Monads::Try[StandardError] do
+                # @type [Mihari::Rule]
+                rule = Services::RuleBuilder.call(path_or_id)
 
-                check_diff_l = ->(rule) { check_diff rule }
-                update_and_call_l = ->(runner) { update_and_call runner }
-
-                result = builder.result.bind(check_diff_l).bind(update_and_call_l)
-
-                alert = result.value!
-                data = Entities::Alert.represent(alert)
-                puts JSON.pretty_generate(data.as_json)
-              end
-            end
-
-            no_commands do
-              #
-              # @param [Mihari::RuleRunner] rule
-              #
-              def check_diff(rule)
                 force_overwrite = options["force_overwrite"] || false
                 message = "There is a diff in the rule. Are you sure you want to overwrite the rule? (y/n)"
-                runner = Services::RuleRunner.new(rule)
-
-                exit 0 if runner.diff? && !force_overwrite && !yes?(message)
+                exit 0 if rule.diff? && !force_overwrite && !yes?(message)
 
-                Success runner
-              end
+                rule.update_or_create
+                rule.call
+              end.to_result
 
-              #
-              # @param [Mihari::RuleRunner] runner
-              #
-              def update_and_call(runner)
-                Dry::Monads::Try[StandardError] do
-                  runner.update_or_create
-                  runner.call
-                end.to_result
-              end
+              # @type [Mihari::Models::Alert]
+              alert = result.value!
+              data = Entities::Alert.represent(alert)
+              puts JSON.pretty_generate(data.as_json)
             end
           end
         end

data/lib/mihari/constants.rb

@@ -5,7 +5,7 @@ module Mihari
   DEFAULT_DATA_TYPES = Types::DataTypes.values.freeze
 
   # @return [Array<Hash>]
-  DEFAULT_EMITTERS = Emitters::Database.class_keys.map { |name| { emitter: name } }.freeze
+  DEFAULT_EMITTERS = Emitters::Database.class_keys.map { |name| { emitter: name.downcase } }.freeze
 
   # @return [Array<Hash>]
   DEFAULT_ENRICHERS = Mihari.enricher_to_class.keys.map { |name| { enricher: name.downcase } }.freeze

data/lib/mihari/database.rb

@@ -134,7 +134,7 @@ module Mihari
   class Database
     class << self
       #
-      # DB migraration
+      # DB migration
       #
       # @param [Symbol] direction
       #