mihari 5.7.1 → 6.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2d27340b82929d2a9dba6cab93c9e429112eed2f254f18595b765bd65852cdfa
-  data.tar.gz: 137141dad2d1b3c4f5b33a2ed38824a9e19c965c7d13e99d20358c2d546dd24c
+  metadata.gz: 2846f5154fcdde4cda4f0237c79e047fc498b96b5c21a6152287ac9ab11faac9
+  data.tar.gz: f43b4e20a59b1274b62c5e9768f153429540067882508b34f0c36932cc9cb5ef
 SHA512:
-  metadata.gz: 84825a14e57d1e740d7ee1e312ac23454017665b92dbdbe9fd2fe0cb55eb900addda0d9563049cf493094b67e9f1e5888d09fe37e7a102db5732f09837558fec
-  data.tar.gz: 393031f2550d7943271dd4b662c65b1da2df322661cdf1228808848e93f7bb7f65acaf8d1861599036bd951b464b5248f615e2abd738b171699eaf396b64f7c4
+  metadata.gz: b54da0da25e57531c1efef94bdc42df715569b9e88842ca330d1d0893f4c20990bb2ded22a001f31816cbd72f00f43473f0254f222a9849308be746576128e3c
+  data.tar.gz: 7167afd356c4f945e66631bb3c6cefc06678961b480511d85939e6ede9eccec9a5f83a948a169be9ee7ba5cc79af918a447b8f01e5280faff3138071e7808beb

data/config.ru

@@ -1,5 +1,7 @@
 require "./lib/mihari"
 
+require "better_errors"
+
 # set rack env as development
 ENV["RACK_ENV"] ||= "development"
 

data/lib/mihari/analyzers/dnstwister.rb

@@ -25,9 +25,7 @@ module Mihari
         raise ValueError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
 
         domains = client.fuzz(query)
-        Parallel.map(domains) do |domain|
-          resolvable?(domain) ? domain : nil
-        end.compact
+        Parallel.map(domains) { |domain| resolvable?(domain) ? domain : nil }.compact
       end
 
       private
@@ -55,7 +53,7 @@ module Mihari
       def resolvable?(domain)
         Resolv.getaddress domain
         true
-      rescue Resolv::ResolvError => _e
+      rescue Resolv::ResolvError
         false
       end
     end

data/lib/mihari/analyzers/urlscan.rb

@@ -34,10 +34,7 @@ module Mihari
       def artifacts
         # @type [Array<Mihari::Models::Artifact>]
         artifacts = client.search_with_pagination(query, pagination_limit: pagination_limit).map(&:artifacts).flatten
-
-        artifacts.select do |artifact|
-          allowed_data_types.include? artifact.data_type
-        end
+        artifacts.select { |artifact| allowed_data_types.include? artifact.data_type }
       end
 
       def configuration_keys

data/lib/mihari/cli/main.rb

@@ -4,6 +4,8 @@ require "thor"
 require "thor/hollaback"
 
 # Commands
+require "mihari/commands/mixins"
+
 require "mihari/commands/alert"
 require "mihari/commands/database"
 require "mihari/commands/search"
@@ -30,19 +32,9 @@ module Mihari
       include Mihari::Commands::Version
       include Mihari::Commands::Web
 
-      no_commands do
-        def unwrap_error(err)
-          return err unless err.is_a?(Dry::Monads::UnwrapError)
-
-          # NOTE: UnwrapError's receiver can be either of:
-          #       - Dry::Monads::Try::Error
-          #       - Dry::Monads::Result::Failure
-          receiver = err.receiver
-          return receiver.exception if receiver.is_a?(Dry::Monads::Try::Error)
-
-          receiver.failure
-        end
+      include Mihari::Mixins::UnwrapError
 
+      no_commands do
         def safe_execute
           yield
         rescue StandardError => e

data/lib/mihari/clients/base.rb

@@ -41,7 +41,7 @@ module Mihari
       # @return [::HTTP::Client]
       #
       def http
-        HTTP::Factory.build headers: headers, timeout: timeout
+        @http ||= HTTP::Factory.build(headers: headers, timeout: timeout)
       end
 
       #
@@ -63,6 +63,17 @@ module Mihari
         http.get(url_for(path), params: params)
       end
 
+      #
+      # @param [String] path
+      # @param [Hash, nil] params
+      #
+      # @return [Hash]
+      #
+      def get_json(path, params: nil)
+        res = get(path, params: params)
+        JSON.parse res.body.to_s
+      end
+
       #
       # @param [String] path
       # @param [Hash, nil] json
@@ -72,6 +83,17 @@ module Mihari
       def post(path, json: {})
         http.post(url_for(path), json: json)
       end
+
+      #
+      # @param [String] path
+      # @param [Hash, nil] json
+      #
+      # @return [Hash]
+      #
+      def post_json(path, json: {})
+        res = http.post(url_for(path), json: json)
+        JSON.parse res.body.to_s
+      end
     end
   end
 end

data/lib/mihari/clients/binaryedge.rb

@@ -38,9 +38,7 @@ module Mihari
           page: page,
           only_ips: only_ips
         }.compact
-
-        res = get("/query/search", params: params)
-        Structs::BinaryEdge::Response.from_dynamic! JSON.parse(res.body.to_s)
+        Structs::BinaryEdge::Response.from_dynamic! get_json("/query/search", params: params)
       end
 
       #

data/lib/mihari/clients/censys.rb

@@ -46,8 +46,7 @@ module Mihari
       #
       def search(query, per_page: nil, cursor: nil)
         params = { q: query, per_page: per_page, cursor: cursor }.compact
-        res = get("/api/v2/hosts/search", params: params)
-        Structs::Censys::Response.from_dynamic! JSON.parse(res.body.to_s)
+        Structs::Censys::Response.from_dynamic! get_json("/api/v2/hosts/search", params: params)
       end
 
       #

data/lib/mihari/clients/crtsh.rb

@@ -27,9 +27,8 @@ module Mihari
       def search(identity, match: nil, exclude: nil)
         params = { identity: identity, match: match, exclude: exclude, output: "json" }.compact
 
-        res = get("/", params: params)
-        parsed = JSON.parse(res.body.to_s)
-
+        # @type [Array[Hash]]
+        parsed = get_json("/", params: params)
         parsed.map do |result|
           values = result["name_value"].to_s.lines.map(&:chomp)
           values.map { |value| Models::Artifact.new(data: value, metadata: result) }

data/lib/mihari/clients/dnstwister.rb

@@ -23,8 +23,7 @@ module Mihari
       # @return [Array<String>]
       #
       def fuzz(domain)
-        res = get("/api/fuzz/#{to_hex(domain)}")
-        res = JSON.parse(res.body.to_s)
+        res = get_json("/api/fuzz/#{to_hex(domain)}")
         fuzzy_domains = res["fuzzy_domains"] || []
         fuzzy_domains.map { |d| d["domain"] }
       end

data/lib/mihari/clients/fofa.rb

@@ -52,8 +52,7 @@ module Mihari
       def search(query, page:, size: PAGE_SIZE)
         qbase64 = Base64.urlsafe_encode64(query)
         params = { qbase64: qbase64, size: size, page: page, email: email, key: api_key }.compact
-        res = get("/api/v1/search/all", params: params)
-        Structs::Fofa::Response.from_dynamic! JSON.parse(res.body.to_s)
+        Structs::Fofa::Response.from_dynamic! get_json("/api/v1/search/all", params: params)
       end
 
       #
@@ -71,7 +70,6 @@ module Mihari
             y.yield res
 
             break if res.error
-
             break if (res.results || []).length < size
 
             sleep_pagination_interval

data/lib/mihari/clients/greynoise.rb

@@ -39,8 +39,7 @@ module Mihari
       #
       def gnql_search(query, size: PAGE_SIZE, scroll: nil)
         params = { query: query, size: size, scroll: scroll }.compact
-        res = get("/v2/experimental/gnql", params: params)
-        Structs::GreyNoise::Response.from_dynamic! JSON.parse(res.body.to_s)
+        Structs::GreyNoise::Response.from_dynamic! get_json("/v2/experimental/gnql", params: params)
       end
 
       #

data/lib/mihari/clients/hunterhow.rb

@@ -52,8 +52,7 @@ module Mihari
           end_time: end_time,
           "api-key": api_key
         }.compact
-        res = get("/search", params: params)
-        Structs::HunterHow::Response.from_dynamic! JSON.parse(res.body.to_s)
+        Structs::HunterHow::Response.from_dynamic! get_json("/search", params: params)
       end
 
       #

data/lib/mihari/clients/misp.rb

@@ -27,8 +27,7 @@ module Mihari
       # @return [Hash]
       #
       def create_event(payload)
-        res = post("/events/add", json: payload)
-        JSON.parse(res.body.to_s)
+        post_json("/events/add", json: payload)
       end
     end
   end

data/lib/mihari/clients/onyphe.rb

@@ -40,8 +40,7 @@ module Mihari
       #
       def datascan(query, page: 1)
         params = { page: page, apikey: api_key }
-        res = get("/api/v2/simple/datascan/#{query}", params: params)
-        Structs::Onyphe::Response.from_dynamic! JSON.parse(res.body.to_s)
+        Structs::Onyphe::Response.from_dynamic! get_json("/api/v2/simple/datascan/#{query}", params: params)
       end
 
       #

data/lib/mihari/clients/otx.rb

@@ -65,7 +65,7 @@ module Mihari
       # @return [Hash]
       #
       def query_by_ip(ip)
-        _get "/api/v1/indicators/IPv4/#{ip}/passive_dns"
+        get_json "/api/v1/indicators/IPv4/#{ip}/passive_dns"
       end
 
       #
@@ -74,19 +74,7 @@ module Mihari
       # @return [Hash]
       #
       def query_by_domain(domain)
-        _get "/api/v1/indicators/domain/#{domain}/passive_dns"
-      end
-
-      private
-
-      #
-      # @param [String] path
-      #
-      # @return [Hash]
-      #
-      def _get(path)
-        res = get(path)
-        JSON.parse(res.body.to_s)
+        get_json "/api/v1/indicators/domain/#{domain}/passive_dns"
       end
     end
   end

data/lib/mihari/clients/passivetotal.rb

@@ -33,7 +33,7 @@ module Mihari
       #
       def passive_dns_search(query)
         params = { query: query }
-        res = _get("/v2/dns/passive/unique", params: params)
+        res = get_json("/v2/dns/passive/unique", params: params)
         res["results"] || []
       end
 
@@ -49,7 +49,7 @@ module Mihari
           query: query,
           field: "email"
         }.compact
-        res = _get("/v2/whois/search", params: params)
+        res = get_json("/v2/whois/search", params: params)
         results = res["results"] || []
         results.map do |result|
           data = result["domain"]
@@ -66,26 +66,13 @@ module Mihari
       #
       def ssl_search(query)
         params = { query: query }
-        res = _get("/v2/ssl-certificate/history", params: params)
+        res = get_json("/v2/ssl-certificate/history", params: params)
         results = res["results"] || []
         results.map do |result|
           data = result["ipAddresses"]
           data.map { |d| Models::Artifact.new(data: d, metadata: result) }
         end.flatten
       end
-
-      private
-
-      #
-      # @param [String] path
-      # @param [Hash] params
-      #
-      # @return [Hash]
-      #
-      def _get(path, params: {})
-        res = get(path, params: params)
-        JSON.parse(res.body.to_s)
-      end
     end
   end
 end

data/lib/mihari/clients/publsedive.rb

@@ -29,7 +29,7 @@ module Mihari
       # @return [Hash]
       #
       def get_indicator(ip_or_domain)
-        _get "/api/info.php", params: { indicator: ip_or_domain }
+        get_json "/api/info.php", params: { indicator: ip_or_domain, key: api_key }
       end
 
       #
@@ -38,22 +38,7 @@ module Mihari
       # @return [Hash]
       #
       def get_properties(indicator_id)
-        _get "/api/info.php", params: { iid: indicator_id, get: "properties" }
-      end
-
-      private
-
-      #
-      # @param [String] path
-      # @param [Hash] params
-      #
-      # @return [Hash]
-      #
-      def _get(path, params: {})
-        params["key"] = api_key
-
-        res = get(path, params: params)
-        JSON.parse res.body.to_s
+        get_json "/api/info.php", params: { iid: indicator_id, get: "properties", key: api_key }
       end
     end
   end

data/lib/mihari/clients/securitytrails.rb

@@ -70,7 +70,7 @@ module Mihari
       # @return [Array<Hash>]
       #
       def search_by_mail(mail)
-        res = _post "/v1/domains/list", json: { filter: { whois_email: mail } }
+        res = post_json "/v1/domains/list", json: { filter: { whois_email: mail } }
         res["records"] || []
       end
 
@@ -80,7 +80,7 @@ module Mihari
       # @return [Array<Hash>]
       #
       def search_by_ip(ip)
-        res = _post "/v1/domains/list", json: { filter: { ipv4: ip } }
+        res = post_json "/v1/domains/list", json: { filter: { ipv4: ip } }
         res["records"] || []
       end
 
@@ -114,29 +114,7 @@ module Mihari
       # @return [Array<Hash>]
       #
       def get_dns_history(domain, type:, page:)
-        _get "/v1/history/#{domain}/dns/#{type}", params: { page: page }
-      end
-
-      #
-      # @param [String] path
-      # @param [Hash, nil] params
-      #
-      # @return [Hash]
-      #
-      def _get(path, params:)
-        res = get(path, params: params)
-        JSON.parse(res.body.to_s)
-      end
-
-      #
-      # @param [String] path
-      # @param [Hash, nil] json
-      #
-      # @return [Hash]
-      #
-      def _post(path, json:)
-        res = post(path, json: json)
-        JSON.parse(res.body.to_s)
+        get_json "/v1/history/#{domain}/dns/#{type}", params: { page: page }
       end
     end
   end

data/lib/mihari/clients/shodan.rb

@@ -46,8 +46,7 @@ module Mihari
           minify: minify,
           key: api_key
         }
-        res = get("/shodan/host/search", params: params)
-        Structs::Shodan::Response.from_dynamic! JSON.parse(res.body.to_s)
+        Structs::Shodan::Response.from_dynamic! get_json("/shodan/host/search", params: params)
       end
 
       #

data/lib/mihari/clients/the_hive.rb

@@ -29,8 +29,7 @@ module Mihari
       #
       def alert(json)
         json = json.to_camelback_keys.compact
-        res = post("/alert", json: json)
-        JSON.parse(res.body.to_s)
+        post_json("/alert", json: json)
       end
     end
   end

data/lib/mihari/clients/urlscan.rb

@@ -36,8 +36,7 @@ module Mihari
       #
       def search(q, size: nil, search_after: nil)
         params = { q: q, size: size, search_after: search_after }.compact
-        res = get("/api/v1/search/", params: params)
-        Structs::Urlscan::Response.from_dynamic! JSON.parse(res.body.to_s)
+        Structs::Urlscan::Response.from_dynamic! get_json("/api/v1/search/", params: params)
       end
 
       #

data/lib/mihari/clients/virustotal.rb

@@ -33,7 +33,7 @@ module Mihari
       # @return [Hash]
       #
       def domain_search(query)
-        _get("/api/v3/domains/#{query}/resolutions")
+        get_json "/api/v3/domains/#{query}/resolutions"
       end
 
       #
@@ -42,7 +42,7 @@ module Mihari
       # @return [Hash]
       #
       def ip_search(query)
-        _get("/api/v3/ip_addresses/#{query}/resolutions")
+        get_json "/api/v3/ip_addresses/#{query}/resolutions"
       end
 
       #
@@ -53,8 +53,7 @@ module Mihari
       #
       def intel_search(query, cursor: nil)
         params = { query: query, cursor: cursor }.compact
-        res = _get("/api/v3/intelligence/search", params: params)
-        Structs::VirusTotalIntelligence::Response.from_dynamic! res
+        Structs::VirusTotalIntelligence::Response.from_dynamic! get_json("/api/v3/intelligence/search", params: params)
       end
 
       #
@@ -79,19 +78,6 @@ module Mihari
           end
         end
       end
-
-      private
-
-      #
-      # @param [String] path
-      # @param [Hash] params
-      #
-      # @return [Hash]
-      #
-      def _get(path, params: {})
-        res = get(path, params: params)
-        JSON.parse(res.body.to_s)
-      end
     end
   end
 end

data/lib/mihari/clients/zoomeye.rb

@@ -30,6 +30,13 @@ module Mihari
         super(base_url, headers: headers, pagination_interval: pagination_interval, timeout: timeout)
       end
 
+      #
+      # @return [::HTTP::Client]
+      #
+      def http
+        @http ||= HTTP::Factory.build(headers: headers, timeout: timeout, raise_exception: false)
+      end
+
       #
       # Search the Host devices
       #
@@ -45,8 +52,7 @@ module Mihari
           page: page,
           facets: facets
         }.compact
-
-        _get("/host/search", params: params)
+        get_json "/host/search", params: params
       end
 
       #
@@ -88,8 +94,7 @@ module Mihari
           page: page,
           facets: facets
         }.compact
-
-        _get("/web/search", params: params)
+        get_json "/web/search", params: params
       end
 
       #
@@ -115,21 +120,6 @@ module Mihari
           end
         end
       end
-
-      private
-
-      #
-      # @param [String] path
-      # @param [Hash] params
-      #
-      # @return [Hash, nil]
-      #
-      def _get(path, params: {})
-        res = get(path, params: params)
-        JSON.parse(res.body.to_s)
-      rescue HTTPError
-        nil
-      end
     end
   end
 end

data/lib/mihari/commands/alert.rb

@@ -10,24 +10,24 @@ module Mihari
         def included(thor)
           thor.class_eval do
             include Dry::Monads[:result, :try]
+            include Mixins
 
             desc "add [PATH]", "Add an alert"
+            around :with_db_connection
             #
             # @param [String] path
             #
             def add(path)
-              Mihari::Database.with_db_connection do
-                result = Dry::Monads::Try[StandardError] do
-                  # @type [Mihari::Services::AlertProxy]
-                  proxy = Mihari::Services::AlertBuilder.call(path)
-                  Mihari::Services::AlertRunner.call(proxy)
-                end.to_result
+              result = Dry::Monads::Try[StandardError] do
+                # @type [Mihari::Services::AlertProxy]
+                proxy = Mihari::Services::AlertBuilder.call(path)
+                Mihari::Services::AlertRunner.call proxy
+              end.to_result
 
-                # @type [Mihari::Models::Alert]
-                alert = result.value!
-                data = Entities::Alert.represent(alert)
-                puts JSON.pretty_generate(data.as_json)
-              end
+              # @type [Mihari::Models::Alert]
+              alert = result.value!
+              data = Entities::Alert.represent(alert)
+              puts JSON.pretty_generate(data.as_json)
             end
           end
         end

data/lib/mihari/commands/database.rb

@@ -9,15 +9,17 @@ module Mihari
       class << self
         def included(thor)
           thor.class_eval do
+            include Mixins
+
             desc "migrate", "Migrate DB schemas"
+            around :with_db_connection
             method_option :verbose, type: :boolean, default: true
             #
             # @param [String] direction
             #
             def migrate(direction = "up")
               ActiveRecord::Migration.verbose = options["verbose"]
-
-              Mihari::Database.with_db_connection { Mihari::Database.migrate direction.to_sym }
+              Mihari::Database.migrate direction.to_sym
             end
           end
         end

data/lib/mihari/commands/mixins.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Commands
+    module Mixins
+      def with_db_connection(&block)
+        Mihari::Database.with_db_connection(&block)
+      end
+    end
+  end
+end

data/lib/mihari/commands/search.rb

@@ -10,8 +10,10 @@ module Mihari
         def included(thor)
           thor.class_eval do
             include Dry::Monads[:try, :result]
+            include Mixins
 
             desc "search [PATH_OR_ID]", "Search by a rule (Outputs null if there is no new finding)"
+            around :with_db_connection
             method_option :force_overwrite, type: :boolean, aliases: "-f", desc: "Force overwriting a rule"
             #
             # Search by a rule
@@ -19,24 +21,22 @@ module Mihari
             # @param [String] path_or_id
             #
             def search(path_or_id)
-              Mihari::Database.with_db_connection do
-                result = Dry::Monads::Try[StandardError] do
-                  # @type [Mihari::Rule]
-                  rule = Services::RuleBuilder.call(path_or_id)
+              result = Dry::Monads::Try[StandardError] do
+                # @type [Mihari::Rule]
+                rule = Services::RuleBuilder.call(path_or_id)
 
-                  force_overwrite = options["force_overwrite"] || false
-                  message = "There is a diff in the rule. Are you sure you want to overwrite the rule? (y/n)"
-                  exit 0 if rule.diff? && !force_overwrite && !yes?(message)
+                force_overwrite = options["force_overwrite"] || false
+                message = "There is a diff in the rule. Are you sure you want to overwrite the rule? (y/n)"
+                exit 0 if rule.diff? && !force_overwrite && !yes?(message)
 
-                  rule.update_or_create
-                  rule.call
-                end.to_result
+                rule.update_or_create
+                rule.call
+              end.to_result
 
-                # @type [Mihari::Models::Alert]
-                alert = result.value!
-                data = Entities::Alert.represent(alert)
-                puts JSON.pretty_generate(data.as_json)
-              end
+              # @type [Mihari::Models::Alert]
+              alert = result.value!
+              data = Entities::Alert.represent(alert)
+              puts JSON.pretty_generate(data.as_json)
             end
           end
         end

data/lib/mihari/constants.rb

@@ -5,7 +5,7 @@ module Mihari
   DEFAULT_DATA_TYPES = Types::DataTypes.values.freeze
 
   # @return [Array<Hash>]
-  DEFAULT_EMITTERS = Emitters::Database.class_keys.map { |name| { emitter: name } }.freeze
+  DEFAULT_EMITTERS = Emitters::Database.class_keys.map { |name| { emitter: name.downcase } }.freeze
 
   # @return [Array<Hash>]
   DEFAULT_ENRICHERS = Mihari.enricher_to_class.keys.map { |name| { enricher: name.downcase } }.freeze