mihari 5.0.1 → 5.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b7944fcbb2ef6b1ff7fccbe5c8158bd21a186b05e8fae70a6700256dce10adbb
-  data.tar.gz: 36605153506952b323be6e3a7646fd4446f280943fc6c587d7885e8eec413c33
+  metadata.gz: e19317302956178dc4302543d81a0920018aa384595f91c69dd70110086d575a
+  data.tar.gz: 80d6314c2df13a4a28ec71a0d4b358e74ab0ee9778d818658a997c1fd821f062
 SHA512:
-  metadata.gz: ba53c1fb987ffd933017ccc64a3a72adc032de81a377e34684c4927304d295d85531ad11e796abd3c17489411fcf15eedd494d7ff7a8b7d0c53fdeb511eb5a8d
-  data.tar.gz: 6dff687f32dfef7f0cc19b76cba77a6233ebbefa9b90f522f37092468370eb5abb0e7f185d86f74077944cb0c76609b01390e7878ca494a82419ac44e59d93e5
+  metadata.gz: b54bffc456bc114a2a8b52e5acbcc3f30d7571124fc9431fbda209bb57910eabe7950d86c50ef750c41d5df604a5aa08c9affb25d434db8a4f53d85ba8ae4921
+  data.tar.gz: 4bce535d8d6d2573102b0197984854b84628a08ab37855c2a7f7fb1574b701e6b4ca816d3b23b3719d7cde0b7de89753af3b386d43af39e422c7c06b1d65448c

data/.rspec

@@ -1,3 +1,3 @@
---format documentation
+--format Fuubar
 --color
 --require spec_helper

data/docker/Dockerfile

@@ -3,7 +3,7 @@ FROM ruby:3.1.3-alpine3.17
 RUN apk --no-cache add git build-base ruby-dev sqlite-dev postgresql-dev mysql-client mysql-dev && \
   gem install pg mysql2
 
-ARG MIHARI_VERSION=4.11.0
+ARG MIHARI_VERSION=5.1.0
 
 RUN gem install mihari -v ${MIHARI_VERSION}
 

data/lib/mihari/analyzers/binaryedge.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "binaryedge"
-
 module Mihari
   module Analyzers
     class BinaryEdge < Base
@@ -44,8 +42,8 @@ module Mihari
       # @return [Hash]
       #
       def search_with_page(query, page: 1)
-        api.host.search(query, page: page)
-      rescue ::BinaryEdge::Error => e
+        client.search(query, page: page)
+      rescue UnsuccessfulStatusCodeError => e
         raise RetryableError, e if e.message.include?("Request time limit exceeded")
 
         raise e
@@ -58,7 +56,7 @@ module Mihari
       #
       def search
         responses = []
-        (1..Float::INFINITY).each do |page|
+        (1..500).each do |page|
           res = search_with_page(query, page: page)
           total = res["total"].to_i
 
@@ -75,8 +73,12 @@ module Mihari
         %w[binaryedge_api_key]
       end
 
-      def api
-        @api ||= ::BinaryEdge::API.new(api_key)
+      #
+      #
+      # @return [Mihari::Clients::BinaryEdge]
+      #
+      def client
+        @client ||= Clients::BinaryEdge.new(api_key: api_key)
       end
     end
   end

data/lib/mihari/analyzers/censys.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "censysx"
-
 module Mihari
   module Analyzers
     class Censys < Base
@@ -42,7 +40,7 @@ module Mihari
 
         cursor = nil
         loop do
-          response = api.search(query, cursor: cursor)
+          response = client.search(query, cursor: cursor)
           response = Structs::Censys::Response.from_dynamic!(response)
 
           artifacts << response_to_artifacts(response)
@@ -106,8 +104,8 @@ module Mihari
         %w[censys_id censys_secret]
       end
 
-      def api
-        @api ||= ::Censys::API.new(id, secret)
+      def client
+        @client ||= Clients::Censys.new(id: id, secret: secret)
       end
 
       def id?

data/lib/mihari/analyzers/circl.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "passive_circl"
-
 module Mihari
   module Analyzers
     class CIRCL < Base
@@ -42,8 +40,8 @@ module Mihari
         %w[circl_passive_password circl_passive_username]
       end
 
-      def api
-        @api ||= ::PassiveCIRCL::API.new(username: username, password: password)
+      def client
+        @client ||= Clients::CIRCL.new(username: username, password: password)
       end
 
       #
@@ -68,7 +66,7 @@ module Mihari
       # @return [Array<String>]
       #
       def passive_dns_search
-        results = api.dns.query(@query)
+        results = client.dns_query(@query)
         results.filter_map do |result|
           type = result["rrtype"]
           (type == "A") ? result["rdata"] : nil
@@ -81,7 +79,7 @@ module Mihari
       # @return [Array<String>]
       #
       def passive_ssl_search
-        result = api.ssl.cquery(@query)
+        result = client.ssl_cquery(@query)
         seen = result["seen"] || []
         seen.uniq
       end

data/lib/mihari/analyzers/crtsh.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "crtsh"
-
 module Mihari
   module Analyzers
     class Crtsh < Base
@@ -21,8 +19,11 @@ module Mihari
 
       private
 
-      def api
-        @api ||= ::Crtsh::API.new
+      #
+      # @return [Mihari::Clients::Crtsh]
+      #
+      def client
+        @client ||= Mihari::Clients::Crtsh.new
       end
 
       #
@@ -32,9 +33,7 @@ module Mihari
       #
       def search
         exclude = exclude_expired ? "expired" : nil
-        api.search(query, exclude: exclude)
-      rescue ::Crtsh::Error => _e
-        []
+        client.search(query, exclude: exclude)
       end
     end
   end

data/lib/mihari/analyzers/dnpedia.rb

@@ -1,22 +1,18 @@
 # frozen_string_literal: true
 
-require "dnpedia"
-
 module Mihari
   module Analyzers
     class DNPedia < Base
       param :query
 
-      option :tags, default: proc { [] }
-
       def artifacts
         search || []
       end
 
       private
 
-      def api
-        @api ||= ::DNPedia::API.new
+      def client
+        @client ||= Clients::DNPedia.new
       end
 
       #
@@ -25,7 +21,7 @@ module Mihari
       # @return [Array<Mihari::Artifact>]
       #
       def search
-        res = api.search(query)
+        res = client.search(query)
         rows = res["rows"] || []
         rows.map do |row|
           data = [row["name"], row["zoneid"]].join(".")

data/lib/mihari/analyzers/dnstwister.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "dnstwister"
-
 module Mihari
   module Analyzers
     class DNSTwister < Base
@@ -35,8 +33,8 @@ module Mihari
         type == "domain"
       end
 
-      def api
-        @api ||= ::DNSTwister::API.new
+      def client
+        @client ||= Clients::DNSTwister.new
       end
 
       #
@@ -61,7 +59,7 @@ module Mihari
       def search
         raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
 
-        res = api.fuzz(query)
+        res = client.fuzz(query)
         fuzzy_domains = res["fuzzy_domains"] || []
         domains = fuzzy_domains.map { |domain| domain["domain"] }
         Parallel.map(domains) do |domain|

data/lib/mihari/analyzers/feed.rb

@@ -8,26 +8,28 @@ module Mihari
     class Feed < Base
       param :query
 
-      option :http_request_method, default: proc { "GET" }
-      option :http_request_headers, default: proc { {} }
-      option :http_request_payload, default: proc { {} }
-      option :http_request_payload_type, default: proc {}
+      option :method, default: proc { "GET" }
+      option :headers, default: proc { {} }
+      option :params, default: proc {}
+      option :json, default: proc {}
+      option :data, default: proc {}
 
       option :selector, default: proc { "" }
 
       def artifacts
-        Mihari::Feed::Parser.new(data).parse selector
+        Mihari::Feed::Parser.new(results).parse selector
       end
 
       private
 
-      def data
+      def results
         reader = Mihari::Feed::Reader.new(
           query,
-          http_request_method: http_request_method,
-          http_request_headers: http_request_headers,
-          http_request_payload: http_request_payload,
-          http_request_payload_type: http_request_payload_type
+          method: method,
+          headers: headers,
+          params: params,
+          json: json,
+          data: data
         )
         reader.read
       end

data/lib/mihari/analyzers/greynoise.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "greynoise"
-
 module Mihari
   module Analyzers
     class GreyNoise < Base
@@ -31,8 +29,8 @@ module Mihari
         %w[greynoise_api_key]
       end
 
-      def api
-        @api ||= ::GreyNoise::API.new(key: api_key)
+      def client
+        @client ||= Clients::GreyNoise.new(api_key: api_key)
       end
 
       #
@@ -41,7 +39,7 @@ module Mihari
       # @return [Hash]
       #
       def search
-        api.experimental.gnql(query, size: PAGE_SIZE)
+        client.gnql_search(query, size: PAGE_SIZE)
       end
 
       #

data/lib/mihari/analyzers/onyphe.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "onyphe"
 require "normalize_country"
 
 module Mihari
@@ -37,8 +36,8 @@ module Mihari
         %w[onyphe_api_key]
       end
 
-      def api
-        @api ||= ::Onyphe::API.new(api_key)
+      def client
+        @client ||= Clients::Onyphe.new(api_key: api_key)
       end
 
       #
@@ -50,7 +49,7 @@ module Mihari
       # @return [Structs::Onyphe::Response]
       #
       def search_with_page(query, page: 1)
-        res = api.simple.datascan(query, page: page)
+        res = client.datascan(query, page: page)
         Structs::Onyphe::Response.from_dynamic!(res)
       end
 

data/lib/mihari/analyzers/otx.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "mihari/analyzers/clients/otx"
-
 module Mihari
   module Analyzers
     class OTX < Base
@@ -35,7 +33,7 @@ module Mihari
       end
 
       def client
-        @client ||= Mihari::Analyzers::Clients::OTX.new(api_key)
+        @client ||= Mihari::Clients::OTX.new(api_key: api_key)
       end
 
       #

data/lib/mihari/analyzers/passivetotal.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "passivetotal"
-
 module Mihari
   module Analyzers
     class PassiveTotal < Base
@@ -42,8 +40,8 @@ module Mihari
         %w[passivetotal_username passivetotal_api_key]
       end
 
-      def api
-        @api ||= ::PassiveTotal::API.new(username: username, api_key: api_key)
+      def client
+        @client ||= Clients::PassiveTotal.new(username: username, api_key: api_key)
       end
 
       #
@@ -79,7 +77,7 @@ module Mihari
       # @return [Array<String>]
       #
       def passive_dns_search
-        res = api.dns.passive_unique(query)
+        res = client.passive_dns_search(query)
         res["results"] || []
       end
 
@@ -89,7 +87,7 @@ module Mihari
       # @return [Array<Mihari::Artifact>]
       #
       def reverse_whois_search
-        res = api.whois.search(query: query, field: "email")
+        res = client.reverse_whois_search(query: query, field: "email")
         results = res["results"] || []
         results.map do |result|
           data = result["domain"]
@@ -103,7 +101,7 @@ module Mihari
       # @return [Array<Mihari::Artifact>]
       #
       def ssl_search
-        res = api.ssl.history(query)
+        res = client.ssl_search(query)
         results = res["results"] || []
         results.map do |result|
           data = result["ipAddresses"]

data/lib/mihari/analyzers/pulsedive.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "pulsedive"
-
 module Mihari
   module Analyzers
     class Pulsedive < Base
@@ -34,8 +32,8 @@ module Mihari
         %w[pulsedive_api_key]
       end
 
-      def api
-        @api ||= ::Pulsedive::API.new(api_key)
+      def client
+        @client ||= Clients::PulseDive.new(api_key: api_key)
       end
 
       #
@@ -55,12 +53,12 @@ module Mihari
       def search
         raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
 
-        indicator = api.indicator.get_by_value(query)
+        indicator = client.get_indicator(query)
         iid = indicator["iid"]
 
-        properties = api.indicator.get_properties_by_id(iid)
+        properties = client.get_properties(iid)
         (properties["dns"] || []).filter_map do |property|
-          if ["A", "PTR"].include?(property["name"])
+          if %w[A PTR].include?(property["name"])
             nil
           else
             data = property["value"]

data/lib/mihari/analyzers/shodan.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "shodan"
-
 module Mihari
   module Analyzers
     class Shodan < Base
@@ -37,8 +35,8 @@ module Mihari
         %w[shodan_api_key]
       end
 
-      def api
-        @api ||= ::Shodan::API.new(key: api_key)
+      def client
+        @client ||= Clients::Shodan.new(api_key: api_key)
       end
 
       #
@@ -50,11 +48,7 @@ module Mihari
       # @return [Hash]
       #
       def search_with_page(query, page: 1)
-        api.host.search(query, page: page)
-      rescue ::Shodan::Error => e
-        raise RetryableError, e if e.message.include?("request timed out")
-
-        raise e
+        client.search(query, page: page)
       end
 
       #

data/lib/mihari/analyzers/urlscan.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "urlscan"
-
 module Mihari
   module Analyzers
     class Urlscan < Base
@@ -20,7 +18,10 @@ module Mihari
       def initialize(*args, **kwargs)
         super
 
-        raise InvalidInputError, "allowed_data_types should be any of url, domain and ip." unless valid_alllowed_data_types?
+        unless valid_alllowed_data_types?
+          raise InvalidInputError,
+            "allowed_data_types should be any of url, domain and ip."
+        end
 
         @api_key = kwargs[:api_key] || Mihari.config.urlscan_api_key
       end
@@ -44,8 +45,8 @@ module Mihari
         %w[urlscan_api_key]
       end
 
-      def api
-        @api ||= ::UrlScan::API.new(api_key)
+      def client
+        @client ||= Clients::UrlScan.new(api_key: api_key)
       end
 
       #
@@ -54,7 +55,7 @@ module Mihari
       # @return [Structs::Urlscan::Response]
       #
       def search_with_search_after(search_after: nil)
-        res = api.search(query, size: SIZE, search_after: search_after)
+        res = client.search(query, size: SIZE, search_after: search_after)
         Structs::Urlscan::Response.from_dynamic! res
       end
 

data/lib/mihari/analyzers/virustotal.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "virustotal"
-
 module Mihari
   module Analyzers
     class VirusTotal < Base
@@ -33,8 +31,8 @@ module Mihari
         %w[virustotal_api_key]
       end
 
-      def api
-        @api = ::VirusTotal::API.new(key: api_key)
+      def client
+        @client = Clients::VirusTotal.new(api_key: api_key)
       end
 
       #
@@ -68,7 +66,7 @@ module Mihari
       # @return [Array<Mihari::Artifact>]
       #
       def domain_search
-        res = api.domain.resolutions(query)
+        res = client.domain_search(query)
 
         data = res["data"] || []
         data.filter_map do |item|
@@ -83,7 +81,7 @@ module Mihari
       # @return [Array<Mihari::Artifact>]
       #
       def ip_search
-        res = api.ip_address.resolutions(query)
+        res = client.ip_search(query)
 
         data = res["data"] || []
         data.filter_map do |item|

data/lib/mihari/analyzers/virustotal_intelligence.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "virustotal"
-
 module Mihari
   module Analyzers
     class VirusTotalIntelligence < Base
@@ -40,8 +38,8 @@ module Mihari
       #
       # @return [::VirusTotal::API]
       #
-      def api
-        @api = ::VirusTotal::API.new(key: api_key)
+      def client
+        @client = Clients::VirusTotal.new(api_key: api_key)
       end
 
       #
@@ -54,7 +52,8 @@ module Mihari
         responses = []
 
         loop do
-          response = Structs::VirusTotalIntelligence::Response.from_dynamic!(api.intelligence.search(query, cursor: cursor))
+          response = Structs::VirusTotalIntelligence::Response.from_dynamic!(client.intel_search(query,
+            cursor: cursor))
           responses << response
 
           break if response.meta.cursor.nil?

data/lib/mihari/analyzers/zoomeye.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "zoomeye"
-
 module Mihari
   module Analyzers
     class ZoomEye < Base
@@ -48,8 +46,8 @@ module Mihari
         %w[zoomeye_api_key]
       end
 
-      def api
-        @api ||= ::ZoomEye::API.new(api_key: api_key)
+      def client
+        @client ||= Clients::ZoomEye.new(api_key: api_key)
       end
 
       #
@@ -83,9 +81,7 @@ module Mihari
       # @return [Hash, nil]
       #
       def _host_search(query, page: 1)
-        api.host.search(query, page: page)
-      rescue ::ZoomEye::Error => _e
-        nil
+        client.host_search(query, page: page)
       end
 
       #
@@ -118,9 +114,7 @@ module Mihari
       # @return [Hash, nil]
       #
       def _web_search(query, page: 1)
-        api.web.search(query, page: page)
-      rescue ::ZoomEye::Error => _e
-        nil
+        client.web_search(query, page: page)
       end
 
       #

data/lib/mihari/cli/database.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require "mihari/commands/database"
+
+module Mihari
+  module CLI
+    class Database < Base
+      include Mihari::Commands::Database
+    end
+  end
+end

data/lib/mihari/cli/main.rb

@@ -3,23 +3,29 @@
 require "thor"
 
 # Commands
-require "mihari/commands/initializer"
 require "mihari/commands/searcher"
-require "mihari/commands/validator"
 require "mihari/commands/version"
 require "mihari/commands/web"
+require "mihari/commands/database"
 
 # CLIs
 require "mihari/cli/base"
 
+require "mihari/cli/database"
+require "mihari/cli/rule"
+
 module Mihari
   module CLI
     class Main < Base
       include Mihari::Commands::Searcher
       include Mihari::Commands::Version
       include Mihari::Commands::Web
-      include Mihari::Commands::Validator
-      include Mihari::Commands::Initializer
+
+      desc "db", "Sub commands for DB"
+      subcommand "db", Database
+
+      desc "rule", "Sub commands for rule"
+      subcommand "rule", Rule
     end
   end
 end

data/lib/mihari/cli/rule.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require "mihari/commands/rule"
+
+module Mihari
+  module CLI
+    class Rule < Base
+      include Mihari::Commands::Rule
+    end
+  end
+end