mihari 5.1.0 → 5.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9481229c15756515053bbc91453db90d914e40e5f536d0b5aaad0f2b8fff5868
-  data.tar.gz: 1ea1f9caa3dbb1f945f4a3ba9f9af17c80c786594692de1564ca4f76692fa628
+  metadata.gz: 5415ee0f5bb820e8073383e4771589bbbaa39fc313af3aa434c5754f34bb9056
+  data.tar.gz: a4da4ce859fa718c900572b865518110f92649698a76ca64c55e38ebad2f9856
 SHA512:
-  metadata.gz: f13bb894b69d4c92b03afc30fcce1be95ba1af1b0cb963c553a328fb85f771c852ee5526af751a098dd4f1b69c60592f021a0e3ee7e40004856d1d4eabc53aba
-  data.tar.gz: 751ad7fc17f2db38961e835f8c9b2a325cbf82692df65ef29c638bcf3eec9cbc17905a39bcbcf5a53bcf477adbf6080a23aa40702b3fb5a55727038eefe9b09d
+  metadata.gz: c0b95e672f17d5ba0035624b035da278c863dc0b1a0860e3393cd8f001125964270233da7bf5ffb08c9057889b4a51698829de059089ed4d31a0f4fd7d0aa3e8
+  data.tar.gz: 30cbba0170104212e9244dfeac820a1a223468bae940719c242778d5bffd95b147f482548aebcc2386713fe43cc0c1b97bf2d71d08d318ad62a4b1a707f3e605

data/.gitmodules

@@ -1,3 +0,0 @@
-[submodule "vendor/rbs/gem_rbs_collection"]
-	path = vendor/rbs/gem_rbs_collection
-	url = https://github.com/ruby/gem_rbs_collection.git

data/.rubocop.yml

@@ -2,3 +2,9 @@ Style/HashSyntax:
   EnforcedShorthandSyntax: either
 Style/StringLiterals:
   EnforcedStyle: double_quotes
+Metrics/BlockLength:
+  Exclude:
+    - "spec/**/*"
+    - "*.gemspec"
+Metrics/ClassLength:
+  Enabled: false

data/README.md

@@ -45,7 +45,6 @@ Mihari supports the following services by default.
 - [Censys](http://censys.io)
 - [CIRCL passive DNS](https://www.circl.lu/services/passive-dns/) / [passive SSL](https://www.circl.lu/services/passive-ssl/)
 - [crt.sh](https://crt.sh/)
-- [DN Pedia](https://dnpedia.com/)
 - [dnstwister](https://dnstwister.report/)
 - [GreyNoise](https://www.greynoise.io/)
 - [Onyphe](https://onyphe.io)

data/lib/mihari/analyzers/base.rb

@@ -7,7 +7,6 @@ module Mihari
 
       option :rule, default: proc {}
 
-      include Mixins::AutonomousSystem
       include Mixins::Configurable
       include Mixins::Retriable
 
@@ -20,6 +19,15 @@ module Mihari
         @base_time = Time.now.utc
       end
 
+      #
+      # Load/overwrite rule
+      #
+      # @param [String] path_or_id
+      #
+      def load_rule(path_or_id)
+        @rule = Structs::Rule.from_path_or_id path_or_id
+      end
+
       # @return [Array<String>, Array<Mihari::Artifact>]
       def artifacts
         raise NotImplementedError, "You must implement #{self.class}##{__method__}"
@@ -30,35 +38,41 @@ module Mihari
         self.class.to_s.split("::").last.to_s
       end
 
+      # @return [String]
+      def class_name
+        self.class.to_s.split("::").last
+      end
+
       #
       # Set artifacts & run emitters in parallel
       #
       # @return [Mihari::Alert, nil]
       #
       def run
-        unless configured?
-          class_name = self.class.to_s.split("::").last
-          raise ConfigurationError, "#{class_name} is not configured correctly"
-        end
-
-        set_enriched_artifacts
-
-        responses = Parallel.map(valid_emitters) do |emitter|
-          run_emitter emitter
-        end
+        raise ConfigurationError, "#{class_name} is not configured correctly" unless configured?
 
+        alert_or_something = bulk_emit
         # returns Mihari::Alert created by the database emitter
-        responses.find { |res| res.is_a?(Mihari::Alert) }
+        alert_or_something.find { |res| res.is_a?(Mihari::Alert) }
       end
 
       #
-      # Run emitter
+      # Bulk emit
+      #
+      # @return [Array<Mihari::Alert>]
+      #
+      def bulk_emit
+        Parallel.map(valid_emitters) { |emitter| emit emitter }.compact
+      end
+
+      #
+      # Emit an alert
       #
       # @param [Mihari::Emitters::Base] emitter
       #
       # @return [Mihari::Alert, nil]
       #
-      def run_emitter(emitter)
+      def emit(emitter)
         return if enriched_artifacts.empty?
 
         alert_or_something = emitter.run(artifacts: enriched_artifacts, rule: rule)
@@ -80,6 +94,7 @@ module Mihari
       #
       # Normalize artifacts
       # - Convert data (string) into an artifact
+      # - Set rule ID
       # - Reject an invalid artifact
       # - Uniquefy artifacts by data
       #
@@ -89,17 +104,16 @@ module Mihari
         @normalized_artifacts ||= artifacts.compact.sort.map do |artifact|
           # No need to set data_type manually
           # It is set automatically in #initialize
-          artifact.is_a?(Artifact) ? artifact : Artifact.new(data: artifact, source: source)
-        end.select(&:valid?).uniq(&:data).map do |artifact|
+          artifact = artifact.is_a?(Artifact) ? artifact : Artifact.new(data: artifact, source: source)
           artifact.rule_id = rule&.id
           artifact
-        end
+        end.select(&:valid?).uniq(&:data)
       end
 
       private
 
       #
-      # Uniquefy artifacts
+      # Uniquefy artifacts (assure rule level uniqueness)
       #
       # @return [Array<Mihari::Artifact>]
       #
@@ -121,15 +135,6 @@ module Mihari
         end
       end
 
-      #
-      # Set enriched artifacts
-      #
-      # @return [nil]
-      #
-      def set_enriched_artifacts
-        retry_on_error { enriched_artifacts }
-      end
-
       #
       # Select valid emitters
       #

data/lib/mihari/analyzers/binaryedge.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "binaryedge"
-
 module Mihari
   module Analyzers
     class BinaryEdge < Base
@@ -12,6 +10,12 @@ module Mihari
       # @return [String, nil]
       attr_reader :api_key
 
+      # @return [String]
+      attr_reader :query
+
+      # @return [Integer]
+      attr_reader :interval
+
       def initialize(*args, **kwargs)
         super(*args, **kwargs)
 
@@ -43,9 +47,9 @@ module Mihari
       #
       # @return [Hash]
       #
-      def search_with_page(query, page: 1)
-        api.host.search(query, page: page)
-      rescue ::BinaryEdge::Error => e
+      def search_with_page(page: 1)
+        client.search(query, page: page)
+      rescue UnsuccessfulStatusCodeError => e
         raise RetryableError, e if e.message.include?("Request time limit exceeded")
 
         raise e
@@ -58,8 +62,8 @@ module Mihari
       #
       def search
         responses = []
-        (1..Float::INFINITY).each do |page|
-          res = search_with_page(query, page: page)
+        (1..500).each do |page|
+          res = search_with_page(page: page)
           total = res["total"].to_i
 
           responses << res
@@ -75,8 +79,12 @@ module Mihari
         %w[binaryedge_api_key]
       end
 
-      def api
-        @api ||= ::BinaryEdge::API.new(api_key)
+      #
+      #
+      # @return [Mihari::Clients::BinaryEdge]
+      #
+      def client
+        @client ||= Clients::BinaryEdge.new(api_key: api_key)
       end
     end
   end

data/lib/mihari/analyzers/censys.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "censysx"
-
 module Mihari
   module Analyzers
     class Censys < Base
@@ -15,6 +13,12 @@ module Mihari
       # @return [String, nil]
       attr_reader :secret
 
+      # @return [Integer]
+      attr_reader :interval
+
+      # @return [String]
+      attr_reader :query
+
       def initialize(*args, **kwargs)
         super(*args, **kwargs)
 
@@ -42,11 +46,8 @@ module Mihari
 
         cursor = nil
         loop do
-          response = api.search(query, cursor: cursor)
-          response = Structs::Censys::Response.from_dynamic!(response)
-
-          artifacts << response_to_artifacts(response)
-
+          response = client.search(query, cursor: cursor)
+          artifacts << response.result.to_artifacts(source)
           cursor = response.result.links.next
           break if cursor == ""
 
@@ -57,57 +58,12 @@ module Mihari
         artifacts.flatten.uniq(&:data)
       end
 
-      #
-      # Extract IPv4s from Censys search API response
-      #
-      # @param [Structs::Censys::Response] response
-      #
-      # @return [Array<String>]
-      #
-      def response_to_artifacts(response)
-        response.result.hits.map { |hit| build_artifact(hit) }
-      end
-
-      #
-      # Build an artifact from a Shodan search API response
-      #
-      # @param [Structs::Censys::Hit] hit
-      #
-      # @return [Artifact]
-      #
-      def build_artifact(hit)
-        as = AutonomousSystem.new(asn: normalize_asn(hit.autonomous_system.asn))
-
-        # sometimes Censys overlooks country
-        # then set geolocation as nil
-        geolocation = nil
-        unless hit.location.country.nil?
-          geolocation = Geolocation.new(
-            country: hit.location.country,
-            country_code: hit.location.country_code
-          )
-        end
-
-        ports = hit.services.map(&:port).map do |port|
-          Port.new(port: port)
-        end
-
-        Artifact.new(
-          data: hit.ip,
-          source: source,
-          metadata: hit.metadata,
-          autonomous_system: as,
-          geolocation: geolocation,
-          ports: ports
-        )
-      end
-
       def configuration_keys
         %w[censys_id censys_secret]
       end
 
-      def api
-        @api ||= ::Censys::API.new(id, secret)
+      def client
+        @client ||= Clients::Censys.new(id: id, secret: secret)
       end
 
       def id?

data/lib/mihari/analyzers/circl.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "passive_circl"
-
 module Mihari
   module Analyzers
     class CIRCL < Base
@@ -18,6 +16,9 @@ module Mihari
       # @return [String, nil]
       attr_reader :password
 
+      # @return [String]
+      attr_reader :query
+
       def initialize(*args, **kwargs)
         super
 
@@ -42,8 +43,8 @@ module Mihari
         %w[circl_passive_password circl_passive_username]
       end
 
-      def api
-        @api ||= ::PassiveCIRCL::API.new(username: username, password: password)
+      def client
+        @client ||= Clients::CIRCL.new(username: username, password: password)
       end
 
       #
@@ -68,7 +69,7 @@ module Mihari
       # @return [Array<String>]
       #
       def passive_dns_search
-        results = api.dns.query(@query)
+        results = client.dns_query(query)
         results.filter_map do |result|
           type = result["rrtype"]
           (type == "A") ? result["rdata"] : nil
@@ -81,7 +82,7 @@ module Mihari
       # @return [Array<String>]
       #
       def passive_ssl_search
-        result = api.ssl.cquery(@query)
+        result = client.ssl_cquery(query)
         seen = result["seen"] || []
         seen.uniq
       end

data/lib/mihari/analyzers/crtsh.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "crtsh"
-
 module Mihari
   module Analyzers
     class Crtsh < Base
@@ -9,6 +7,12 @@ module Mihari
 
       option :exclude_expired, default: proc { true }
 
+      # @return [Boolean]
+      attr_reader :exclude_expired
+
+      # @return [String]
+      attr_reader :query
+
       def artifacts
         results = search
         results.map do |result|
@@ -21,8 +25,11 @@ module Mihari
 
       private
 
-      def api
-        @api ||= ::Crtsh::API.new
+      #
+      # @return [Mihari::Clients::Crtsh]
+      #
+      def client
+        @client ||= Mihari::Clients::Crtsh.new
       end
 
       #
@@ -32,9 +39,7 @@ module Mihari
       #
       def search
         exclude = exclude_expired ? "expired" : nil
-        api.search(query, exclude: exclude)
-      rescue ::Crtsh::Error => _e
-        []
+        client.search(query, exclude: exclude)
       end
     end
   end

data/lib/mihari/analyzers/dnstwister.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "dnstwister"
-
 module Mihari
   module Analyzers
     class DNSTwister < Base
@@ -9,10 +7,12 @@ module Mihari
 
       param :query
 
-      option :tags, default: proc { [] }
-
+      # @return [String]
       attr_reader :type
 
+      # @return [String]
+      attr_reader :query
+
       def initialize(*args, **kwargs)
         super
 
@@ -35,8 +35,8 @@ module Mihari
         type == "domain"
       end
 
-      def api
-        @api ||= ::DNSTwister::API.new
+      def client
+        @client ||= Clients::DNSTwister.new
       end
 
       #
@@ -61,7 +61,7 @@ module Mihari
       def search
         raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
 
-        res = api.fuzz(query)
+        res = client.fuzz(query)
         fuzzy_domains = res["fuzzy_domains"] || []
         domains = fuzzy_domains.map { |domain| domain["domain"] }
         Parallel.map(domains) do |domain|

data/lib/mihari/analyzers/feed.rb

@@ -8,26 +8,49 @@ module Mihari
     class Feed < Base
       param :query
 
-      option :http_request_method, default: proc { "GET" }
-      option :http_request_headers, default: proc { {} }
-      option :http_request_payload, default: proc { {} }
-      option :http_request_payload_type, default: proc {}
+      option :method, default: proc { "GET" }
+      option :headers, default: proc { {} }
+      option :params, default: proc {}
+      option :json, default: proc {}
+      option :data, default: proc {}
 
       option :selector, default: proc { "" }
 
+      # @return [Hash, nil]
+      attr_reader :data
+
+      # @return [Hash, nil]
+      attr_reader :json
+
+      # @return [Hash, nil]
+      attr_reader :params
+
+      # @return [Hash, nil]
+      attr_reader :headers
+
+      # @return [String]
+      attr_reader :method
+
+      # @return [String]
+      attr_reader :selector
+
+      # @return [String]
+      attr_reader :query
+
       def artifacts
-        Mihari::Feed::Parser.new(data).parse selector
+        Mihari::Feed::Parser.new(results).parse selector
       end
 
       private
 
-      def data
+      def results
         reader = Mihari::Feed::Reader.new(
           query,
-          http_request_method: http_request_method,
-          http_request_headers: http_request_headers,
-          http_request_payload: http_request_payload,
-          http_request_payload_type: http_request_payload_type
+          method: method,
+          headers: headers,
+          params: params,
+          json: json,
+          data: data
         )
         reader.read
       end

data/lib/mihari/analyzers/greynoise.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "greynoise"
-
 module Mihari
   module Analyzers
     class GreyNoise < Base
@@ -10,6 +8,9 @@ module Mihari
       # @return [String, nil]
       attr_reader :api_key
 
+      # @return [String]
+      attr_reader :query
+
       def initialize(*args, **kwargs)
         super(*args, **kwargs)
 
@@ -17,10 +18,8 @@ module Mihari
       end
 
       def artifacts
-        res = Structs::GreyNoise::Response.from_dynamic!(search)
-        res.data.map do |datum|
-          build_artifact datum
-        end
+        res = search
+        res.to_artifacts
       end
 
       private
@@ -31,8 +30,8 @@ module Mihari
         %w[greynoise_api_key]
       end
 
-      def api
-        @api ||= ::GreyNoise::API.new(key: api_key)
+      def client
+        @client ||= Clients::GreyNoise.new(api_key: api_key)
       end
 
       #
@@ -41,31 +40,7 @@ module Mihari
       # @return [Hash]
       #
       def search
-        api.experimental.gnql(query, size: PAGE_SIZE)
-      end
-
-      #
-      # Build an artifact from a GreyNoise search API response
-      #
-      # @param [Structs::GreyNoise::Datum] datum
-      #
-      # @return [Artifact]
-      #
-      def build_artifact(datum)
-        as = AutonomousSystem.new(asn: normalize_asn(datum.metadata.asn))
-
-        geolocation = Geolocation.new(
-          country: datum.metadata.country,
-          country_code: datum.metadata.country_code
-        )
-
-        Artifact.new(
-          data: datum.ip,
-          source: source,
-          metadata: datum.metadata_,
-          autonomous_system: as,
-          geolocation: geolocation
-        )
+        client.gnql_search(query, size: PAGE_SIZE)
       end
     end
   end

data/lib/mihari/analyzers/onyphe.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "onyphe"
 require "normalize_country"
 
 module Mihari
@@ -13,6 +12,12 @@ module Mihari
       # @return [String, nil]
       attr_reader :api_key
 
+      # @return [String]
+      attr_reader :query
+
+      # @return [Integer]
+      attr_reader :interval
+
       def initialize(*args, **kwargs)
         super(*args, **kwargs)
 
@@ -23,10 +28,7 @@ module Mihari
         responses = search
         return [] unless responses
 
-        results = responses.map(&:results).flatten
-        results.map do |result|
-          build_artifact result
-        end
+        responses.map { |response| response.to_artifacts(source) }.flatten
       end
 
       private
@@ -37,8 +39,8 @@ module Mihari
         %w[onyphe_api_key]
       end
 
-      def api
-        @api ||= ::Onyphe::API.new(api_key)
+      def client
+        @client ||= Clients::Onyphe.new(api_key: api_key)
       end
 
       #
@@ -50,8 +52,7 @@ module Mihari
       # @return [Structs::Onyphe::Response]
       #
       def search_with_page(query, page: 1)
-        res = api.simple.datascan(query, page: page)
-        Structs::Onyphe::Response.from_dynamic!(res)
+        client.datascan(query, page: page)
       end
 
       #
@@ -73,33 +74,6 @@ module Mihari
         end
         responses
       end
-
-      #
-      # Build an artifact from an Onyphe search API result
-      #
-      # @param [Structs::Onyphe::Result] result
-      #
-      # @return [Artifact]
-      #
-      def build_artifact(result)
-        as = AutonomousSystem.new(asn: normalize_asn(result.asn))
-
-        geolocation = nil
-        unless result.country_code.nil?
-          geolocation = Geolocation.new(
-            country: NormalizeCountry(result.country_code, to: :short),
-            country_code: result.country_code
-          )
-        end
-
-        Artifact.new(
-          data: result.ip,
-          source: source,
-          metadata: result.metadata,
-          autonomous_system: as,
-          geolocation: geolocation
-        )
-      end
     end
   end
 end

data/lib/mihari/analyzers/otx.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "mihari/analyzers/clients/otx"
-
 module Mihari
   module Analyzers
     class OTX < Base
@@ -15,6 +13,9 @@ module Mihari
       # @return [String, nil]
       attr_reader :api_key
 
+      # @return [String]
+      attr_reader :query
+
       def initialize(*args, **kwargs)
         super
 
@@ -35,7 +36,7 @@ module Mihari
       end
 
       def client
-        @client ||= Mihari::Analyzers::Clients::OTX.new(api_key)
+        @client ||= Mihari::Clients::OTX.new(api_key: api_key)
       end
 
       #