mihari 5.7.1 → 5.7.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2d27340b82929d2a9dba6cab93c9e429112eed2f254f18595b765bd65852cdfa
-  data.tar.gz: 137141dad2d1b3c4f5b33a2ed38824a9e19c965c7d13e99d20358c2d546dd24c
+  metadata.gz: 5c7fa4de39ec815c08930311d4ed642857236021301508c47d298cba157dd811
+  data.tar.gz: 1598a9992be6739957bc5145c11d93f8a869588d42985799d527703178f78a8f
 SHA512:
-  metadata.gz: 84825a14e57d1e740d7ee1e312ac23454017665b92dbdbe9fd2fe0cb55eb900addda0d9563049cf493094b67e9f1e5888d09fe37e7a102db5732f09837558fec
-  data.tar.gz: 393031f2550d7943271dd4b662c65b1da2df322661cdf1228808848e93f7bb7f65acaf8d1861599036bd951b464b5248f615e2abd738b171699eaf396b64f7c4
+  metadata.gz: 6c96715d51778d724b4df1c502d3083c4af084997019cc7ef43fc5d131cfeb02c4b0f06d6353c0d8e75e26dec0fb20038fa36718f9a65bb1a3a771224442bc3e
+  data.tar.gz: 655c0efa77ff2602fef509b351ebf02ab9c222728ddfcfc0da36ba8fa27a4e1518f1d70406b5ba3ace86012c196954a98c9af10a4de58c5dcdf73bbc760d2056

data/lib/mihari/cli/main.rb

@@ -4,6 +4,8 @@ require "thor"
 require "thor/hollaback"
 
 # Commands
+require "mihari/commands/mixins"
+
 require "mihari/commands/alert"
 require "mihari/commands/database"
 require "mihari/commands/search"

data/lib/mihari/clients/base.rb

@@ -41,7 +41,7 @@ module Mihari
       # @return [::HTTP::Client]
       #
       def http
-        HTTP::Factory.build headers: headers, timeout: timeout
+        @http ||= HTTP::Factory.build(headers: headers, timeout: timeout)
       end
 
       #
@@ -63,6 +63,17 @@ module Mihari
         http.get(url_for(path), params: params)
       end
 
+      #
+      # @param [String] path
+      # @param [Hash, nil] params
+      #
+      # @return [Hash]
+      #
+      def get_json(path, params: nil)
+        res = get(path, params: params)
+        JSON.parse res.body.to_s
+      end
+
       #
       # @param [String] path
       # @param [Hash, nil] json
@@ -72,6 +83,17 @@ module Mihari
       def post(path, json: {})
         http.post(url_for(path), json: json)
       end
+
+      #
+      # @param [String] path
+      # @param [Hash, nil] json
+      #
+      # @return [Hash]
+      #
+      def post_json(path, json: {})
+        res = http.post(url_for(path), json: json)
+        JSON.parse res.body.to_s
+      end
     end
   end
 end

data/lib/mihari/clients/binaryedge.rb

@@ -38,9 +38,7 @@ module Mihari
           page: page,
           only_ips: only_ips
         }.compact
-
-        res = get("/query/search", params: params)
-        Structs::BinaryEdge::Response.from_dynamic! JSON.parse(res.body.to_s)
+        Structs::BinaryEdge::Response.from_dynamic! get_json("/query/search", params: params)
       end
 
       #

data/lib/mihari/clients/censys.rb

@@ -46,8 +46,7 @@ module Mihari
       #
       def search(query, per_page: nil, cursor: nil)
         params = { q: query, per_page: per_page, cursor: cursor }.compact
-        res = get("/api/v2/hosts/search", params: params)
-        Structs::Censys::Response.from_dynamic! JSON.parse(res.body.to_s)
+        Structs::Censys::Response.from_dynamic! get_json("/api/v2/hosts/search", params: params)
       end
 
       #

data/lib/mihari/clients/crtsh.rb

@@ -27,9 +27,8 @@ module Mihari
       def search(identity, match: nil, exclude: nil)
         params = { identity: identity, match: match, exclude: exclude, output: "json" }.compact
 
-        res = get("/", params: params)
-        parsed = JSON.parse(res.body.to_s)
-
+        # @type [Array[Hash]]
+        parsed = get_json("/", params: params)
         parsed.map do |result|
           values = result["name_value"].to_s.lines.map(&:chomp)
           values.map { |value| Models::Artifact.new(data: value, metadata: result) }

data/lib/mihari/clients/dnstwister.rb

@@ -23,8 +23,7 @@ module Mihari
       # @return [Array<String>]
       #
       def fuzz(domain)
-        res = get("/api/fuzz/#{to_hex(domain)}")
-        res = JSON.parse(res.body.to_s)
+        res = get_json("/api/fuzz/#{to_hex(domain)}")
         fuzzy_domains = res["fuzzy_domains"] || []
         fuzzy_domains.map { |d| d["domain"] }
       end

data/lib/mihari/clients/fofa.rb

@@ -52,8 +52,7 @@ module Mihari
       def search(query, page:, size: PAGE_SIZE)
         qbase64 = Base64.urlsafe_encode64(query)
         params = { qbase64: qbase64, size: size, page: page, email: email, key: api_key }.compact
-        res = get("/api/v1/search/all", params: params)
-        Structs::Fofa::Response.from_dynamic! JSON.parse(res.body.to_s)
+        Structs::Fofa::Response.from_dynamic! get_json("/api/v1/search/all", params: params)
       end
 
       #
@@ -71,7 +70,6 @@ module Mihari
             y.yield res
 
             break if res.error
-
             break if (res.results || []).length < size
 
             sleep_pagination_interval

data/lib/mihari/clients/greynoise.rb

@@ -39,8 +39,7 @@ module Mihari
       #
       def gnql_search(query, size: PAGE_SIZE, scroll: nil)
         params = { query: query, size: size, scroll: scroll }.compact
-        res = get("/v2/experimental/gnql", params: params)
-        Structs::GreyNoise::Response.from_dynamic! JSON.parse(res.body.to_s)
+        Structs::GreyNoise::Response.from_dynamic! get_json("/v2/experimental/gnql", params: params)
       end
 
       #

data/lib/mihari/clients/hunterhow.rb

@@ -52,8 +52,7 @@ module Mihari
           end_time: end_time,
           "api-key": api_key
         }.compact
-        res = get("/search", params: params)
-        Structs::HunterHow::Response.from_dynamic! JSON.parse(res.body.to_s)
+        Structs::HunterHow::Response.from_dynamic! get_json("/search", params: params)
       end
 
       #

data/lib/mihari/clients/misp.rb

@@ -27,8 +27,7 @@ module Mihari
       # @return [Hash]
       #
       def create_event(payload)
-        res = post("/events/add", json: payload)
-        JSON.parse(res.body.to_s)
+        post_json("/events/add", json: payload)
       end
     end
   end

data/lib/mihari/clients/onyphe.rb

@@ -40,8 +40,7 @@ module Mihari
       #
       def datascan(query, page: 1)
         params = { page: page, apikey: api_key }
-        res = get("/api/v2/simple/datascan/#{query}", params: params)
-        Structs::Onyphe::Response.from_dynamic! JSON.parse(res.body.to_s)
+        Structs::Onyphe::Response.from_dynamic! get_json("/api/v2/simple/datascan/#{query}", params: params)
       end
 
       #

data/lib/mihari/clients/otx.rb

@@ -65,7 +65,7 @@ module Mihari
       # @return [Hash]
       #
       def query_by_ip(ip)
-        _get "/api/v1/indicators/IPv4/#{ip}/passive_dns"
+        get_json "/api/v1/indicators/IPv4/#{ip}/passive_dns"
       end
 
       #
@@ -74,19 +74,7 @@ module Mihari
       # @return [Hash]
       #
       def query_by_domain(domain)
-        _get "/api/v1/indicators/domain/#{domain}/passive_dns"
-      end
-
-      private
-
-      #
-      # @param [String] path
-      #
-      # @return [Hash]
-      #
-      def _get(path)
-        res = get(path)
-        JSON.parse(res.body.to_s)
+        get_json "/api/v1/indicators/domain/#{domain}/passive_dns"
       end
     end
   end

data/lib/mihari/clients/passivetotal.rb

@@ -33,7 +33,7 @@ module Mihari
       #
       def passive_dns_search(query)
         params = { query: query }
-        res = _get("/v2/dns/passive/unique", params: params)
+        res = get_json("/v2/dns/passive/unique", params: params)
         res["results"] || []
       end
 
@@ -49,7 +49,7 @@ module Mihari
           query: query,
           field: "email"
         }.compact
-        res = _get("/v2/whois/search", params: params)
+        res = get_json("/v2/whois/search", params: params)
         results = res["results"] || []
         results.map do |result|
           data = result["domain"]
@@ -66,26 +66,13 @@ module Mihari
       #
       def ssl_search(query)
         params = { query: query }
-        res = _get("/v2/ssl-certificate/history", params: params)
+        res = get_json("/v2/ssl-certificate/history", params: params)
         results = res["results"] || []
         results.map do |result|
           data = result["ipAddresses"]
           data.map { |d| Models::Artifact.new(data: d, metadata: result) }
         end.flatten
       end
-
-      private
-
-      #
-      # @param [String] path
-      # @param [Hash] params
-      #
-      # @return [Hash]
-      #
-      def _get(path, params: {})
-        res = get(path, params: params)
-        JSON.parse(res.body.to_s)
-      end
     end
   end
 end

data/lib/mihari/clients/publsedive.rb

@@ -29,7 +29,7 @@ module Mihari
       # @return [Hash]
       #
       def get_indicator(ip_or_domain)
-        _get "/api/info.php", params: { indicator: ip_or_domain }
+        get_json "/api/info.php", params: { indicator: ip_or_domain, key: api_key }
       end
 
       #
@@ -38,22 +38,7 @@ module Mihari
       # @return [Hash]
       #
       def get_properties(indicator_id)
-        _get "/api/info.php", params: { iid: indicator_id, get: "properties" }
-      end
-
-      private
-
-      #
-      # @param [String] path
-      # @param [Hash] params
-      #
-      # @return [Hash]
-      #
-      def _get(path, params: {})
-        params["key"] = api_key
-
-        res = get(path, params: params)
-        JSON.parse res.body.to_s
+        get_json "/api/info.php", params: { iid: indicator_id, get: "properties", key: api_key }
       end
     end
   end

data/lib/mihari/clients/securitytrails.rb

@@ -70,7 +70,7 @@ module Mihari
       # @return [Array<Hash>]
       #
       def search_by_mail(mail)
-        res = _post "/v1/domains/list", json: { filter: { whois_email: mail } }
+        res = post_json "/v1/domains/list", json: { filter: { whois_email: mail } }
         res["records"] || []
       end
 
@@ -80,7 +80,7 @@ module Mihari
       # @return [Array<Hash>]
       #
       def search_by_ip(ip)
-        res = _post "/v1/domains/list", json: { filter: { ipv4: ip } }
+        res = post_json "/v1/domains/list", json: { filter: { ipv4: ip } }
         res["records"] || []
       end
 
@@ -114,29 +114,7 @@ module Mihari
       # @return [Array<Hash>]
       #
       def get_dns_history(domain, type:, page:)
-        _get "/v1/history/#{domain}/dns/#{type}", params: { page: page }
-      end
-
-      #
-      # @param [String] path
-      # @param [Hash, nil] params
-      #
-      # @return [Hash]
-      #
-      def _get(path, params:)
-        res = get(path, params: params)
-        JSON.parse(res.body.to_s)
-      end
-
-      #
-      # @param [String] path
-      # @param [Hash, nil] json
-      #
-      # @return [Hash]
-      #
-      def _post(path, json:)
-        res = post(path, json: json)
-        JSON.parse(res.body.to_s)
+        get_json "/v1/history/#{domain}/dns/#{type}", params: { page: page }
       end
     end
   end

data/lib/mihari/clients/shodan.rb

@@ -46,8 +46,7 @@ module Mihari
           minify: minify,
           key: api_key
         }
-        res = get("/shodan/host/search", params: params)
-        Structs::Shodan::Response.from_dynamic! JSON.parse(res.body.to_s)
+        Structs::Shodan::Response.from_dynamic! get_json("/shodan/host/search", params: params)
       end
 
       #

data/lib/mihari/clients/the_hive.rb

@@ -29,8 +29,7 @@ module Mihari
       #
       def alert(json)
         json = json.to_camelback_keys.compact
-        res = post("/alert", json: json)
-        JSON.parse(res.body.to_s)
+        post_json("/alert", json: json)
       end
     end
   end

data/lib/mihari/clients/urlscan.rb

@@ -36,8 +36,7 @@ module Mihari
       #
       def search(q, size: nil, search_after: nil)
         params = { q: q, size: size, search_after: search_after }.compact
-        res = get("/api/v1/search/", params: params)
-        Structs::Urlscan::Response.from_dynamic! JSON.parse(res.body.to_s)
+        Structs::Urlscan::Response.from_dynamic! get_json("/api/v1/search/", params: params)
       end
 
       #

data/lib/mihari/clients/virustotal.rb

@@ -33,7 +33,7 @@ module Mihari
       # @return [Hash]
       #
       def domain_search(query)
-        _get("/api/v3/domains/#{query}/resolutions")
+        get_json "/api/v3/domains/#{query}/resolutions"
       end
 
       #
@@ -42,7 +42,7 @@ module Mihari
       # @return [Hash]
       #
       def ip_search(query)
-        _get("/api/v3/ip_addresses/#{query}/resolutions")
+        get_json "/api/v3/ip_addresses/#{query}/resolutions"
       end
 
       #
@@ -53,8 +53,7 @@ module Mihari
       #
       def intel_search(query, cursor: nil)
         params = { query: query, cursor: cursor }.compact
-        res = _get("/api/v3/intelligence/search", params: params)
-        Structs::VirusTotalIntelligence::Response.from_dynamic! res
+        Structs::VirusTotalIntelligence::Response.from_dynamic! get_json("/api/v3/intelligence/search", params: params)
       end
 
       #
@@ -79,19 +78,6 @@ module Mihari
           end
         end
       end
-
-      private
-
-      #
-      # @param [String] path
-      # @param [Hash] params
-      #
-      # @return [Hash]
-      #
-      def _get(path, params: {})
-        res = get(path, params: params)
-        JSON.parse(res.body.to_s)
-      end
     end
   end
 end

data/lib/mihari/clients/zoomeye.rb

@@ -30,6 +30,13 @@ module Mihari
         super(base_url, headers: headers, pagination_interval: pagination_interval, timeout: timeout)
       end
 
+      #
+      # @return [::HTTP::Client]
+      #
+      def http
+        @http ||= HTTP::Factory.build(headers: headers, timeout: timeout, raise_exception: false)
+      end
+
       #
       # Search the Host devices
       #
@@ -45,8 +52,7 @@ module Mihari
           page: page,
           facets: facets
         }.compact
-
-        _get("/host/search", params: params)
+        get_json "/host/search", params: params
       end
 
       #
@@ -88,8 +94,7 @@ module Mihari
           page: page,
           facets: facets
         }.compact
-
-        _get("/web/search", params: params)
+        get_json "/web/search", params: params
       end
 
       #
@@ -115,21 +120,6 @@ module Mihari
           end
         end
       end
-
-      private
-
-      #
-      # @param [String] path
-      # @param [Hash] params
-      #
-      # @return [Hash, nil]
-      #
-      def _get(path, params: {})
-        res = get(path, params: params)
-        JSON.parse(res.body.to_s)
-      rescue HTTPError
-        nil
-      end
     end
   end
 end

data/lib/mihari/commands/alert.rb

@@ -10,24 +10,24 @@ module Mihari
         def included(thor)
           thor.class_eval do
             include Dry::Monads[:result, :try]
+            include Mixins
 
             desc "add [PATH]", "Add an alert"
+            around :with_db_connection
             #
             # @param [String] path
             #
             def add(path)
-              Mihari::Database.with_db_connection do
-                result = Dry::Monads::Try[StandardError] do
-                  # @type [Mihari::Services::AlertProxy]
-                  proxy = Mihari::Services::AlertBuilder.call(path)
-                  Mihari::Services::AlertRunner.call(proxy)
-                end.to_result
+              result = Dry::Monads::Try[StandardError] do
+                # @type [Mihari::Services::AlertProxy]
+                proxy = Mihari::Services::AlertBuilder.call(path)
+                Mihari::Services::AlertRunner.call proxy
+              end.to_result
 
-                # @type [Mihari::Models::Alert]
-                alert = result.value!
-                data = Entities::Alert.represent(alert)
-                puts JSON.pretty_generate(data.as_json)
-              end
+              # @type [Mihari::Models::Alert]
+              alert = result.value!
+              data = Entities::Alert.represent(alert)
+              puts JSON.pretty_generate(data.as_json)
             end
           end
         end

data/lib/mihari/commands/database.rb

@@ -9,7 +9,10 @@ module Mihari
       class << self
         def included(thor)
           thor.class_eval do
+            include Mixins
+
             desc "migrate", "Migrate DB schemas"
+            around :with_db_connection
             method_option :verbose, type: :boolean, default: true
             #
             # @param [String] direction
@@ -17,7 +20,7 @@ module Mihari
             def migrate(direction = "up")
               ActiveRecord::Migration.verbose = options["verbose"]
 
-              Mihari::Database.with_db_connection { Mihari::Database.migrate direction.to_sym }
+              Mihari::Database.migrate direction.to_sym
             end
           end
         end

data/lib/mihari/commands/mixins.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Commands
+    module Mixins
+      def with_db_connection(&block)
+        Mihari::Database.with_db_connection(&block)
+      end
+    end
+  end
+end

data/lib/mihari/commands/search.rb

@@ -10,8 +10,10 @@ module Mihari
         def included(thor)
           thor.class_eval do
             include Dry::Monads[:try, :result]
+            include Mixins
 
             desc "search [PATH_OR_ID]", "Search by a rule (Outputs null if there is no new finding)"
+            around :with_db_connection
             method_option :force_overwrite, type: :boolean, aliases: "-f", desc: "Force overwriting a rule"
             #
             # Search by a rule
@@ -19,24 +21,22 @@ module Mihari
             # @param [String] path_or_id
             #
             def search(path_or_id)
-              Mihari::Database.with_db_connection do
-                result = Dry::Monads::Try[StandardError] do
-                  # @type [Mihari::Rule]
-                  rule = Services::RuleBuilder.call(path_or_id)
+              result = Dry::Monads::Try[StandardError] do
+                # @type [Mihari::Rule]
+                rule = Services::RuleBuilder.call(path_or_id)
 
-                  force_overwrite = options["force_overwrite"] || false
-                  message = "There is a diff in the rule. Are you sure you want to overwrite the rule? (y/n)"
-                  exit 0 if rule.diff? && !force_overwrite && !yes?(message)
+                force_overwrite = options["force_overwrite"] || false
+                message = "There is a diff in the rule. Are you sure you want to overwrite the rule? (y/n)"
+                exit 0 if rule.diff? && !force_overwrite && !yes?(message)
 
-                  rule.update_or_create
-                  rule.call
-                end.to_result
+                rule.update_or_create
+                rule.call
+              end.to_result
 
-                # @type [Mihari::Models::Alert]
-                alert = result.value!
-                data = Entities::Alert.represent(alert)
-                puts JSON.pretty_generate(data.as_json)
-              end
+              # @type [Mihari::Models::Alert]
+              alert = result.value!
+              data = Entities::Alert.represent(alert)
+              puts JSON.pretty_generate(data.as_json)
             end
           end
         end

data/lib/mihari/constants.rb

@@ -5,7 +5,7 @@ module Mihari
   DEFAULT_DATA_TYPES = Types::DataTypes.values.freeze
 
   # @return [Array<Hash>]
-  DEFAULT_EMITTERS = Emitters::Database.class_keys.map { |name| { emitter: name } }.freeze
+  DEFAULT_EMITTERS = Emitters::Database.class_keys.map { |name| { emitter: name.downcase } }.freeze
 
   # @return [Array<Hash>]
   DEFAULT_ENRICHERS = Mihari.enricher_to_class.keys.map { |name| { enricher: name.downcase } }.freeze

data/lib/mihari/database.rb

@@ -134,7 +134,7 @@ module Mihari
   class Database
     class << self
       #
-      # DB migraration
+      # DB migration
       #
       # @param [Symbol] direction
       #

data/lib/mihari/http.rb

@@ -9,14 +9,13 @@ module Mihari
     #
     class BetterError < ::HTTP::Feature
       def wrap_response(response)
-        unless response.status.success?
-          raise StatusCodeError.new(
-            "Unsuccessful response code returned: #{response.code}",
-            response.code,
-            response.body.to_s
-          )
-        end
-        response
+        return response if response.status.success?
+
+        raise StatusCodeError.new(
+          "Unsuccessful response code returned: #{response.code}",
+          response.code,
+          response.body.to_s
+        )
       end
 
       def on_error(_request, error)
@@ -35,12 +34,15 @@ module Mihari
         #
         # @param [Integer, nil] timeout
         # @param [Hash] headers
+        # @param [Boolean] raise_exception
         #
         # @return [::HTTP::Client]
         #
-        def build(headers: {}, timeout: nil)
-          client = ::HTTP.use(:better_error).headers(headers)
-          client.timeout(timeout) unless timeout.nil?
+        # @param [Object] raise_exception
+        def build(headers: {}, timeout: nil, raise_exception: true)
+          client = raise_exception ? ::HTTP.use(:better_error) : ::HTTP
+          client = client.headers(headers)
+          client = client.timeout(timeout) unless timeout.nil?
           client
         end
       end

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "5.7.1"
+  VERSION = "5.7.2"
 end