mihari 5.1.1 → 5.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e19317302956178dc4302543d81a0920018aa384595f91c69dd70110086d575a
-  data.tar.gz: 80d6314c2df13a4a28ec71a0d4b358e74ab0ee9778d818658a997c1fd821f062
+  metadata.gz: 5415ee0f5bb820e8073383e4771589bbbaa39fc313af3aa434c5754f34bb9056
+  data.tar.gz: a4da4ce859fa718c900572b865518110f92649698a76ca64c55e38ebad2f9856
 SHA512:
-  metadata.gz: b54bffc456bc114a2a8b52e5acbcc3f30d7571124fc9431fbda209bb57910eabe7950d86c50ef750c41d5df604a5aa08c9affb25d434db8a4f53d85ba8ae4921
-  data.tar.gz: 4bce535d8d6d2573102b0197984854b84628a08ab37855c2a7f7fb1574b701e6b4ca816d3b23b3719d7cde0b7de89753af3b386d43af39e422c7c06b1d65448c
+  metadata.gz: c0b95e672f17d5ba0035624b035da278c863dc0b1a0860e3393cd8f001125964270233da7bf5ffb08c9057889b4a51698829de059089ed4d31a0f4fd7d0aa3e8
+  data.tar.gz: 30cbba0170104212e9244dfeac820a1a223468bae940719c242778d5bffd95b147f482548aebcc2386713fe43cc0c1b97bf2d71d08d318ad62a4b1a707f3e605

data/.gitmodules

@@ -1,3 +0,0 @@
-[submodule "vendor/rbs/gem_rbs_collection"]
-	path = vendor/rbs/gem_rbs_collection
-	url = https://github.com/ruby/gem_rbs_collection.git

data/.rubocop.yml

@@ -2,3 +2,9 @@ Style/HashSyntax:
   EnforcedShorthandSyntax: either
 Style/StringLiterals:
   EnforcedStyle: double_quotes
+Metrics/BlockLength:
+  Exclude:
+    - "spec/**/*"
+    - "*.gemspec"
+Metrics/ClassLength:
+  Enabled: false

data/README.md

@@ -45,7 +45,6 @@ Mihari supports the following services by default.
 - [Censys](http://censys.io)
 - [CIRCL passive DNS](https://www.circl.lu/services/passive-dns/) / [passive SSL](https://www.circl.lu/services/passive-ssl/)
 - [crt.sh](https://crt.sh/)
-- [DN Pedia](https://dnpedia.com/)
 - [dnstwister](https://dnstwister.report/)
 - [GreyNoise](https://www.greynoise.io/)
 - [Onyphe](https://onyphe.io)

data/lib/mihari/analyzers/base.rb

@@ -7,7 +7,6 @@ module Mihari
 
       option :rule, default: proc {}
 
-      include Mixins::AutonomousSystem
       include Mixins::Configurable
       include Mixins::Retriable
 
@@ -20,6 +19,15 @@ module Mihari
         @base_time = Time.now.utc
       end
 
+      #
+      # Load/overwrite rule
+      #
+      # @param [String] path_or_id
+      #
+      def load_rule(path_or_id)
+        @rule = Structs::Rule.from_path_or_id path_or_id
+      end
+
       # @return [Array<String>, Array<Mihari::Artifact>]
       def artifacts
         raise NotImplementedError, "You must implement #{self.class}##{__method__}"
@@ -30,35 +38,41 @@ module Mihari
         self.class.to_s.split("::").last.to_s
       end
 
+      # @return [String]
+      def class_name
+        self.class.to_s.split("::").last
+      end
+
       #
       # Set artifacts & run emitters in parallel
       #
       # @return [Mihari::Alert, nil]
       #
       def run
-        unless configured?
-          class_name = self.class.to_s.split("::").last
-          raise ConfigurationError, "#{class_name} is not configured correctly"
-        end
-
-        set_enriched_artifacts
-
-        responses = Parallel.map(valid_emitters) do |emitter|
-          run_emitter emitter
-        end
+        raise ConfigurationError, "#{class_name} is not configured correctly" unless configured?
 
+        alert_or_something = bulk_emit
         # returns Mihari::Alert created by the database emitter
-        responses.find { |res| res.is_a?(Mihari::Alert) }
+        alert_or_something.find { |res| res.is_a?(Mihari::Alert) }
       end
 
       #
-      # Run emitter
+      # Bulk emit
+      #
+      # @return [Array<Mihari::Alert>]
+      #
+      def bulk_emit
+        Parallel.map(valid_emitters) { |emitter| emit emitter }.compact
+      end
+
+      #
+      # Emit an alert
       #
       # @param [Mihari::Emitters::Base] emitter
       #
       # @return [Mihari::Alert, nil]
       #
-      def run_emitter(emitter)
+      def emit(emitter)
         return if enriched_artifacts.empty?
 
         alert_or_something = emitter.run(artifacts: enriched_artifacts, rule: rule)
@@ -80,6 +94,7 @@ module Mihari
       #
       # Normalize artifacts
       # - Convert data (string) into an artifact
+      # - Set rule ID
       # - Reject an invalid artifact
       # - Uniquefy artifacts by data
       #
@@ -89,17 +104,16 @@ module Mihari
         @normalized_artifacts ||= artifacts.compact.sort.map do |artifact|
           # No need to set data_type manually
           # It is set automatically in #initialize
-          artifact.is_a?(Artifact) ? artifact : Artifact.new(data: artifact, source: source)
-        end.select(&:valid?).uniq(&:data).map do |artifact|
+          artifact = artifact.is_a?(Artifact) ? artifact : Artifact.new(data: artifact, source: source)
           artifact.rule_id = rule&.id
           artifact
-        end
+        end.select(&:valid?).uniq(&:data)
       end
 
       private
 
       #
-      # Uniquefy artifacts
+      # Uniquefy artifacts (assure rule level uniqueness)
       #
       # @return [Array<Mihari::Artifact>]
       #
@@ -121,15 +135,6 @@ module Mihari
         end
       end
 
-      #
-      # Set enriched artifacts
-      #
-      # @return [nil]
-      #
-      def set_enriched_artifacts
-        retry_on_error { enriched_artifacts }
-      end
-
       #
       # Select valid emitters
       #

data/lib/mihari/analyzers/binaryedge.rb

@@ -10,6 +10,12 @@ module Mihari
       # @return [String, nil]
       attr_reader :api_key
 
+      # @return [String]
+      attr_reader :query
+
+      # @return [Integer]
+      attr_reader :interval
+
       def initialize(*args, **kwargs)
         super(*args, **kwargs)
 
@@ -41,7 +47,7 @@ module Mihari
       #
       # @return [Hash]
       #
-      def search_with_page(query, page: 1)
+      def search_with_page(page: 1)
         client.search(query, page: page)
       rescue UnsuccessfulStatusCodeError => e
         raise RetryableError, e if e.message.include?("Request time limit exceeded")
@@ -57,7 +63,7 @@ module Mihari
       def search
         responses = []
         (1..500).each do |page|
-          res = search_with_page(query, page: page)
+          res = search_with_page(page: page)
           total = res["total"].to_i
 
           responses << res

data/lib/mihari/analyzers/censys.rb

@@ -13,6 +13,12 @@ module Mihari
       # @return [String, nil]
       attr_reader :secret
 
+      # @return [Integer]
+      attr_reader :interval
+
+      # @return [String]
+      attr_reader :query
+
       def initialize(*args, **kwargs)
         super(*args, **kwargs)
 
@@ -41,10 +47,7 @@ module Mihari
         cursor = nil
         loop do
           response = client.search(query, cursor: cursor)
-          response = Structs::Censys::Response.from_dynamic!(response)
-
-          artifacts << response_to_artifacts(response)
-
+          artifacts << response.result.to_artifacts(source)
           cursor = response.result.links.next
           break if cursor == ""
 
@@ -55,51 +58,6 @@ module Mihari
         artifacts.flatten.uniq(&:data)
       end
 
-      #
-      # Extract IPv4s from Censys search API response
-      #
-      # @param [Structs::Censys::Response] response
-      #
-      # @return [Array<String>]
-      #
-      def response_to_artifacts(response)
-        response.result.hits.map { |hit| build_artifact(hit) }
-      end
-
-      #
-      # Build an artifact from a Shodan search API response
-      #
-      # @param [Structs::Censys::Hit] hit
-      #
-      # @return [Artifact]
-      #
-      def build_artifact(hit)
-        as = AutonomousSystem.new(asn: normalize_asn(hit.autonomous_system.asn))
-
-        # sometimes Censys overlooks country
-        # then set geolocation as nil
-        geolocation = nil
-        unless hit.location.country.nil?
-          geolocation = Geolocation.new(
-            country: hit.location.country,
-            country_code: hit.location.country_code
-          )
-        end
-
-        ports = hit.services.map(&:port).map do |port|
-          Port.new(port: port)
-        end
-
-        Artifact.new(
-          data: hit.ip,
-          source: source,
-          metadata: hit.metadata,
-          autonomous_system: as,
-          geolocation: geolocation,
-          ports: ports
-        )
-      end
-
       def configuration_keys
         %w[censys_id censys_secret]
       end

data/lib/mihari/analyzers/circl.rb

@@ -16,6 +16,9 @@ module Mihari
       # @return [String, nil]
       attr_reader :password
 
+      # @return [String]
+      attr_reader :query
+
       def initialize(*args, **kwargs)
         super
 
@@ -66,7 +69,7 @@ module Mihari
       # @return [Array<String>]
       #
       def passive_dns_search
-        results = client.dns_query(@query)
+        results = client.dns_query(query)
         results.filter_map do |result|
           type = result["rrtype"]
           (type == "A") ? result["rdata"] : nil
@@ -79,7 +82,7 @@ module Mihari
       # @return [Array<String>]
       #
       def passive_ssl_search
-        result = client.ssl_cquery(@query)
+        result = client.ssl_cquery(query)
         seen = result["seen"] || []
         seen.uniq
       end

data/lib/mihari/analyzers/crtsh.rb

@@ -7,6 +7,12 @@ module Mihari
 
       option :exclude_expired, default: proc { true }
 
+      # @return [Boolean]
+      attr_reader :exclude_expired
+
+      # @return [String]
+      attr_reader :query
+
       def artifacts
         results = search
         results.map do |result|

data/lib/mihari/analyzers/dnstwister.rb

@@ -7,10 +7,12 @@ module Mihari
 
       param :query
 
-      option :tags, default: proc { [] }
-
+      # @return [String]
       attr_reader :type
 
+      # @return [String]
+      attr_reader :query
+
       def initialize(*args, **kwargs)
         super
 

data/lib/mihari/analyzers/feed.rb

@@ -16,6 +16,27 @@ module Mihari
 
       option :selector, default: proc { "" }
 
+      # @return [Hash, nil]
+      attr_reader :data
+
+      # @return [Hash, nil]
+      attr_reader :json
+
+      # @return [Hash, nil]
+      attr_reader :params
+
+      # @return [Hash, nil]
+      attr_reader :headers
+
+      # @return [String]
+      attr_reader :method
+
+      # @return [String]
+      attr_reader :selector
+
+      # @return [String]
+      attr_reader :query
+
       def artifacts
         Mihari::Feed::Parser.new(results).parse selector
       end

data/lib/mihari/analyzers/greynoise.rb

@@ -8,6 +8,9 @@ module Mihari
       # @return [String, nil]
       attr_reader :api_key
 
+      # @return [String]
+      attr_reader :query
+
       def initialize(*args, **kwargs)
         super(*args, **kwargs)
 
@@ -15,10 +18,8 @@ module Mihari
       end
 
       def artifacts
-        res = Structs::GreyNoise::Response.from_dynamic!(search)
-        res.data.map do |datum|
-          build_artifact datum
-        end
+        res = search
+        res.to_artifacts
       end
 
       private
@@ -41,30 +42,6 @@ module Mihari
       def search
         client.gnql_search(query, size: PAGE_SIZE)
       end
-
-      #
-      # Build an artifact from a GreyNoise search API response
-      #
-      # @param [Structs::GreyNoise::Datum] datum
-      #
-      # @return [Artifact]
-      #
-      def build_artifact(datum)
-        as = AutonomousSystem.new(asn: normalize_asn(datum.metadata.asn))
-
-        geolocation = Geolocation.new(
-          country: datum.metadata.country,
-          country_code: datum.metadata.country_code
-        )
-
-        Artifact.new(
-          data: datum.ip,
-          source: source,
-          metadata: datum.metadata_,
-          autonomous_system: as,
-          geolocation: geolocation
-        )
-      end
     end
   end
 end

data/lib/mihari/analyzers/onyphe.rb

@@ -12,6 +12,12 @@ module Mihari
       # @return [String, nil]
       attr_reader :api_key
 
+      # @return [String]
+      attr_reader :query
+
+      # @return [Integer]
+      attr_reader :interval
+
       def initialize(*args, **kwargs)
         super(*args, **kwargs)
 
@@ -22,10 +28,7 @@ module Mihari
         responses = search
         return [] unless responses
 
-        results = responses.map(&:results).flatten
-        results.map do |result|
-          build_artifact result
-        end
+        responses.map { |response| response.to_artifacts(source) }.flatten
       end
 
       private
@@ -49,8 +52,7 @@ module Mihari
       # @return [Structs::Onyphe::Response]
       #
       def search_with_page(query, page: 1)
-        res = client.datascan(query, page: page)
-        Structs::Onyphe::Response.from_dynamic!(res)
+        client.datascan(query, page: page)
       end
 
       #
@@ -72,33 +74,6 @@ module Mihari
         end
         responses
       end
-
-      #
-      # Build an artifact from an Onyphe search API result
-      #
-      # @param [Structs::Onyphe::Result] result
-      #
-      # @return [Artifact]
-      #
-      def build_artifact(result)
-        as = AutonomousSystem.new(asn: normalize_asn(result.asn))
-
-        geolocation = nil
-        unless result.country_code.nil?
-          geolocation = Geolocation.new(
-            country: NormalizeCountry(result.country_code, to: :short),
-            country_code: result.country_code
-          )
-        end
-
-        Artifact.new(
-          data: result.ip,
-          source: source,
-          metadata: result.metadata,
-          autonomous_system: as,
-          geolocation: geolocation
-        )
-      end
     end
   end
 end

data/lib/mihari/analyzers/otx.rb

@@ -13,6 +13,9 @@ module Mihari
       # @return [String, nil]
       attr_reader :api_key
 
+      # @return [String]
+      attr_reader :query
+
       def initialize(*args, **kwargs)
         super
 

data/lib/mihari/analyzers/passivetotal.rb

@@ -16,6 +16,9 @@ module Mihari
       # @return [String, nil]
       attr_reader :api_key
 
+      # @return [String]
+      attr_reader :query
+
       def initialize(*args, **kwargs)
         super(*args, **kwargs)
 

data/lib/mihari/analyzers/pulsedive.rb

@@ -13,6 +13,9 @@ module Mihari
       # @return [String, nil]
       attr_reader :api_key
 
+      # @return [Integer]
+      attr_reader :query
+
       def initialize(*args, **kwargs)
         super
 

data/lib/mihari/analyzers/rule.rb

@@ -7,7 +7,6 @@ module Mihari
       "censys" => Censys,
       "circl" => CIRCL,
       "crtsh" => Crtsh,
-      "dnpedia" => DNPedia,
       "dnstwister" => DNSTwister,
       "feed" => Feed,
       "greynoise" => GreyNoise,

data/lib/mihari/analyzers/securitytrails.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "securitytrails"
-
 module Mihari
   module Analyzers
     class SecurityTrails < Base
@@ -15,6 +13,9 @@ module Mihari
       # @return [String, nil]
       attr_reader :api_key
 
+      # @return [String]
+      attr_reader :query
+
       def initialize(*args, **kwargs)
         super
 
@@ -34,8 +35,8 @@ module Mihari
         %w[securitytrails_api_key]
       end
 
-      def api
-        @api ||= ::SecurityTrails::API.new(api_key)
+      def client
+        @client ||= Clients::SecurityTrails.new(api_key: api_key)
       end
 
       #
@@ -71,8 +72,7 @@ module Mihari
       # @return [Array<String>]
       #
       def domain_search
-        result = api.history.get_all_dns_history(query, type: "a")
-        records = result["records"] || []
+        records = client.get_all_dns_history(query, type: "a")
         records.map do |record|
           (record["values"] || []).map { |value| value["ip"] }
         end.flatten.compact.uniq
@@ -84,8 +84,7 @@ module Mihari
       # @return [Array<Mihari::Artifact>]
       #
       def ip_search
-        result = api.domains.search(filter: { ipv4: query })
-        records = result["records"] || []
+        records = client.search_by_ip(query)
         records.filter_map do |record|
           data = record["hostname"]
           Artifact.new(data: data, source: source, metadata: record)
@@ -98,8 +97,7 @@ module Mihari
       # @return [Array<String>]
       #
       def mail_search
-        result = api.domains.search(filter: { whois_email: query })
-        records = result["records"] || []
+        records = client.search_by_mail(query)
         records.filter_map do |record|
           data = record["hostname"]
           Artifact.new(data: data, source: source, metadata: record)