mihari 5.1.1 → 5.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e19317302956178dc4302543d81a0920018aa384595f91c69dd70110086d575a
-  data.tar.gz: 80d6314c2df13a4a28ec71a0d4b358e74ab0ee9778d818658a997c1fd821f062
+  metadata.gz: dd278975834369b3274b38b590f12d4f2fed97d7626363ff3cfbc9845195b386
+  data.tar.gz: 864ee1deae0bdb0390d9e330eb76c993860c37189dce0d23c9793b012f338275
 SHA512:
-  metadata.gz: b54bffc456bc114a2a8b52e5acbcc3f30d7571124fc9431fbda209bb57910eabe7950d86c50ef750c41d5df604a5aa08c9affb25d434db8a4f53d85ba8ae4921
-  data.tar.gz: 4bce535d8d6d2573102b0197984854b84628a08ab37855c2a7f7fb1574b701e6b4ca816d3b23b3719d7cde0b7de89753af3b386d43af39e422c7c06b1d65448c
+  metadata.gz: 258731713400cb40f6da8f24e6ce0213b883eeff98a22a16a88c84568633ad02ad3e18f4e70e6faea428a29bdcdc257b6ec3c86ebea7979065ec2e42ab445c47
+  data.tar.gz: 6c8beea0c3f4aa42f8fb8217a05c6256a6a8ea10daceb05d05ed2884d8b53c86195c765fa70cf35ea19c41d959fd6df114fa2b609893e828def028f5b16b5fc4

data/.gitmodules

@@ -1,3 +0,0 @@
-[submodule "vendor/rbs/gem_rbs_collection"]
-	path = vendor/rbs/gem_rbs_collection
-	url = https://github.com/ruby/gem_rbs_collection.git

data/.rubocop.yml

@@ -2,3 +2,9 @@ Style/HashSyntax:
   EnforcedShorthandSyntax: either
 Style/StringLiterals:
   EnforcedStyle: double_quotes
+Metrics/BlockLength:
+  Exclude:
+    - "spec/**/*"
+    - "*.gemspec"
+Metrics/ClassLength:
+  Enabled: false

data/README.md

@@ -45,7 +45,6 @@ Mihari supports the following services by default.
 - [Censys](http://censys.io)
 - [CIRCL passive DNS](https://www.circl.lu/services/passive-dns/) / [passive SSL](https://www.circl.lu/services/passive-ssl/)
 - [crt.sh](https://crt.sh/)
-- [DN Pedia](https://dnpedia.com/)
 - [dnstwister](https://dnstwister.report/)
 - [GreyNoise](https://www.greynoise.io/)
 - [Onyphe](https://onyphe.io)

data/lib/mihari/analyzers/base.rb

@@ -7,7 +7,6 @@ module Mihari
 
       option :rule, default: proc {}
 
-      include Mixins::AutonomousSystem
       include Mixins::Configurable
       include Mixins::Retriable
 
@@ -20,6 +19,15 @@ module Mihari
         @base_time = Time.now.utc
       end
 
+      #
+      # Load/overwrite rule
+      #
+      # @param [String] path_or_id
+      #
+      def load_rule(path_or_id)
+        @rule = Structs::Rule.from_path_or_id path_or_id
+      end
+
       # @return [Array<String>, Array<Mihari::Artifact>]
       def artifacts
         raise NotImplementedError, "You must implement #{self.class}##{__method__}"
@@ -30,35 +38,41 @@ module Mihari
         self.class.to_s.split("::").last.to_s
       end
 
+      # @return [String]
+      def class_name
+        self.class.to_s.split("::").last
+      end
+
       #
       # Set artifacts & run emitters in parallel
       #
       # @return [Mihari::Alert, nil]
       #
       def run
-        unless configured?
-          class_name = self.class.to_s.split("::").last
-          raise ConfigurationError, "#{class_name} is not configured correctly"
-        end
-
-        set_enriched_artifacts
-
-        responses = Parallel.map(valid_emitters) do |emitter|
-          run_emitter emitter
-        end
+        raise ConfigurationError, "#{class_name} is not configured correctly" unless configured?
 
+        alert_or_something = bulk_emit
         # returns Mihari::Alert created by the database emitter
-        responses.find { |res| res.is_a?(Mihari::Alert) }
+        alert_or_something.find { |res| res.is_a?(Mihari::Alert) }
       end
 
       #
-      # Run emitter
+      # Bulk emit
+      #
+      # @return [Array<Mihari::Alert>]
+      #
+      def bulk_emit
+        Parallel.map(valid_emitters) { |emitter| emit emitter }.compact
+      end
+
+      #
+      # Emit an alert
       #
       # @param [Mihari::Emitters::Base] emitter
       #
       # @return [Mihari::Alert, nil]
       #
-      def run_emitter(emitter)
+      def emit(emitter)
         return if enriched_artifacts.empty?
 
         alert_or_something = emitter.run(artifacts: enriched_artifacts, rule: rule)
@@ -80,6 +94,7 @@ module Mihari
       #
       # Normalize artifacts
       # - Convert data (string) into an artifact
+      # - Set rule ID
       # - Reject an invalid artifact
       # - Uniquefy artifacts by data
       #
@@ -89,17 +104,16 @@ module Mihari
         @normalized_artifacts ||= artifacts.compact.sort.map do |artifact|
           # No need to set data_type manually
           # It is set automatically in #initialize
-          artifact.is_a?(Artifact) ? artifact : Artifact.new(data: artifact, source: source)
-        end.select(&:valid?).uniq(&:data).map do |artifact|
+          artifact = artifact.is_a?(Artifact) ? artifact : Artifact.new(data: artifact, source: source)
           artifact.rule_id = rule&.id
           artifact
-        end
+        end.select(&:valid?).uniq(&:data)
       end
 
       private
 
       #
-      # Uniquefy artifacts
+      # Uniquefy artifacts (assure rule level uniqueness)
       #
       # @return [Array<Mihari::Artifact>]
       #
@@ -121,15 +135,6 @@ module Mihari
         end
       end
 
-      #
-      # Set enriched artifacts
-      #
-      # @return [nil]
-      #
-      def set_enriched_artifacts
-        retry_on_error { enriched_artifacts }
-      end
-
       #
       # Select valid emitters
       #

data/lib/mihari/analyzers/binaryedge.rb

@@ -10,6 +10,12 @@ module Mihari
       # @return [String, nil]
       attr_reader :api_key
 
+      # @return [String]
+      attr_reader :query
+
+      # @return [Integer]
+      attr_reader :interval
+
       def initialize(*args, **kwargs)
         super(*args, **kwargs)
 
@@ -41,7 +47,7 @@ module Mihari
       #
       # @return [Hash]
       #
-      def search_with_page(query, page: 1)
+      def search_with_page(page: 1)
         client.search(query, page: page)
       rescue UnsuccessfulStatusCodeError => e
         raise RetryableError, e if e.message.include?("Request time limit exceeded")
@@ -57,7 +63,7 @@ module Mihari
       def search
         responses = []
         (1..500).each do |page|
-          res = search_with_page(query, page: page)
+          res = search_with_page(page: page)
           total = res["total"].to_i
 
           responses << res

data/lib/mihari/analyzers/censys.rb

@@ -13,6 +13,12 @@ module Mihari
       # @return [String, nil]
       attr_reader :secret
 
+      # @return [Integer]
+      attr_reader :interval
+
+      # @return [String]
+      attr_reader :query
+
       def initialize(*args, **kwargs)
         super(*args, **kwargs)
 
@@ -21,30 +27,12 @@ module Mihari
       end
 
       def artifacts
-        search
-      end
-
-      def configured?
-        configuration_keys.all? { |key| Mihari.config.send(key) } || (id? && secret?)
-      end
-
-      private
-
-      #
-      # Search
-      #
-      # @return [Array<String>]
-      #
-      def search
         artifacts = []
 
         cursor = nil
         loop do
           response = client.search(query, cursor: cursor)
-          response = Structs::Censys::Response.from_dynamic!(response)
-
-          artifacts << response_to_artifacts(response)
-
+          artifacts << response.result.to_artifacts(source)
           cursor = response.result.links.next
           break if cursor == ""
 
@@ -55,50 +43,11 @@ module Mihari
         artifacts.flatten.uniq(&:data)
       end
 
-      #
-      # Extract IPv4s from Censys search API response
-      #
-      # @param [Structs::Censys::Response] response
-      #
-      # @return [Array<String>]
-      #
-      def response_to_artifacts(response)
-        response.result.hits.map { |hit| build_artifact(hit) }
+      def configured?
+        configuration_keys.all? { |key| Mihari.config.send(key) } || (id? && secret?)
       end
 
-      #
-      # Build an artifact from a Shodan search API response
-      #
-      # @param [Structs::Censys::Hit] hit
-      #
-      # @return [Artifact]
-      #
-      def build_artifact(hit)
-        as = AutonomousSystem.new(asn: normalize_asn(hit.autonomous_system.asn))
-
-        # sometimes Censys overlooks country
-        # then set geolocation as nil
-        geolocation = nil
-        unless hit.location.country.nil?
-          geolocation = Geolocation.new(
-            country: hit.location.country,
-            country_code: hit.location.country_code
-          )
-        end
-
-        ports = hit.services.map(&:port).map do |port|
-          Port.new(port: port)
-        end
-
-        Artifact.new(
-          data: hit.ip,
-          source: source,
-          metadata: hit.metadata,
-          autonomous_system: as,
-          geolocation: geolocation,
-          ports: ports
-        )
-      end
+      private
 
       def configuration_keys
         %w[censys_id censys_secret]

data/lib/mihari/analyzers/circl.rb

@@ -16,6 +16,9 @@ module Mihari
       # @return [String, nil]
       attr_reader :password
 
+      # @return [String]
+      attr_reader :query
+
       def initialize(*args, **kwargs)
         super
 
@@ -27,7 +30,14 @@ module Mihari
       end
 
       def artifacts
-        search || []
+        case type
+        when "domain"
+          passive_dns_search
+        when "hash"
+          passive_ssl_search
+        else
+          raise InvalidInputError, "#{@query}(type: #{@type || "unknown"}) is not supported."
+        end
       end
 
       def configured?
@@ -44,29 +54,13 @@ module Mihari
         @client ||= Clients::CIRCL.new(username: username, password: password)
       end
 
-      #
-      # Passive DNS/SSL search
-      #
-      # @return [Array<String>]
-      #
-      def search
-        case @type
-        when "domain"
-          passive_dns_search
-        when "hash"
-          passive_ssl_search
-        else
-          raise InvalidInputError, "#{@query}(type: #{@type || "unknown"}) is not supported."
-        end
-      end
-
       #
       # Passive DNS search
       #
       # @return [Array<String>]
       #
       def passive_dns_search
-        results = client.dns_query(@query)
+        results = client.dns_query(query)
         results.filter_map do |result|
           type = result["rrtype"]
           (type == "A") ? result["rdata"] : nil
@@ -79,7 +73,7 @@ module Mihari
       # @return [Array<String>]
       #
       def passive_ssl_search
-        result = client.ssl_cquery(@query)
+        result = client.ssl_cquery(query)
         seen = result["seen"] || []
         seen.uniq
       end

data/lib/mihari/analyzers/crtsh.rb

@@ -7,6 +7,12 @@ module Mihari
 
       option :exclude_expired, default: proc { true }
 
+      # @return [Boolean]
+      attr_reader :exclude_expired
+
+      # @return [String]
+      attr_reader :query
+
       def artifacts
         results = search
         results.map do |result|

data/lib/mihari/analyzers/dnstwister.rb

@@ -7,10 +7,12 @@ module Mihari
 
       param :query
 
-      option :tags, default: proc { [] }
-
+      # @return [String]
       attr_reader :type
 
+      # @return [String]
+      attr_reader :query
+
       def initialize(*args, **kwargs)
         super
 
@@ -19,7 +21,14 @@ module Mihari
       end
 
       def artifacts
-        search || []
+        raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
+
+        res = client.fuzz(query)
+        fuzzy_domains = res["fuzzy_domains"] || []
+        domains = fuzzy_domains.map { |domain| domain["domain"] }
+        Parallel.map(domains) do |domain|
+          resolvable?(domain) ? domain : nil
+        end.compact
       end
 
       private
@@ -50,22 +59,6 @@ module Mihari
       rescue Resolv::ResolvError => _e
         false
       end
-
-      #
-      # Search
-      #
-      # @return [Array<String>]
-      #
-      def search
-        raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
-
-        res = client.fuzz(query)
-        fuzzy_domains = res["fuzzy_domains"] || []
-        domains = fuzzy_domains.map { |domain| domain["domain"] }
-        Parallel.map(domains) do |domain|
-          resolvable?(domain) ? domain : nil
-        end.compact
-      end
     end
   end
 end

data/lib/mihari/analyzers/feed.rb

@@ -16,6 +16,27 @@ module Mihari
 
       option :selector, default: proc { "" }
 
+      # @return [Hash, nil]
+      attr_reader :data
+
+      # @return [Hash, nil]
+      attr_reader :json
+
+      # @return [Hash, nil]
+      attr_reader :params
+
+      # @return [Hash, nil]
+      attr_reader :headers
+
+      # @return [String]
+      attr_reader :method
+
+      # @return [String]
+      attr_reader :selector
+
+      # @return [String]
+      attr_reader :query
+
       def artifacts
         Mihari::Feed::Parser.new(results).parse selector
       end

data/lib/mihari/analyzers/greynoise.rb

@@ -8,6 +8,9 @@ module Mihari
       # @return [String, nil]
       attr_reader :api_key
 
+      # @return [String]
+      attr_reader :query
+
       def initialize(*args, **kwargs)
         super(*args, **kwargs)
 
@@ -15,10 +18,8 @@ module Mihari
       end
 
       def artifacts
-        res = Structs::GreyNoise::Response.from_dynamic!(search)
-        res.data.map do |datum|
-          build_artifact datum
-        end
+        res = search
+        res.to_artifacts
       end
 
       private
@@ -41,30 +42,6 @@ module Mihari
       def search
         client.gnql_search(query, size: PAGE_SIZE)
       end
-
-      #
-      # Build an artifact from a GreyNoise search API response
-      #
-      # @param [Structs::GreyNoise::Datum] datum
-      #
-      # @return [Artifact]
-      #
-      def build_artifact(datum)
-        as = AutonomousSystem.new(asn: normalize_asn(datum.metadata.asn))
-
-        geolocation = Geolocation.new(
-          country: datum.metadata.country,
-          country_code: datum.metadata.country_code
-        )
-
-        Artifact.new(
-          data: datum.ip,
-          source: source,
-          metadata: datum.metadata_,
-          autonomous_system: as,
-          geolocation: geolocation
-        )
-      end
     end
   end
 end

data/lib/mihari/analyzers/onyphe.rb

@@ -12,6 +12,12 @@ module Mihari
       # @return [String, nil]
       attr_reader :api_key
 
+      # @return [String]
+      attr_reader :query
+
+      # @return [Integer]
+      attr_reader :interval
+
       def initialize(*args, **kwargs)
         super(*args, **kwargs)
 
@@ -22,10 +28,7 @@ module Mihari
         responses = search
         return [] unless responses
 
-        results = responses.map(&:results).flatten
-        results.map do |result|
-          build_artifact result
-        end
+        responses.map { |response| response.to_artifacts(source) }.flatten
       end
 
       private
@@ -49,8 +52,7 @@ module Mihari
       # @return [Structs::Onyphe::Response]
       #
       def search_with_page(query, page: 1)
-        res = client.datascan(query, page: page)
-        Structs::Onyphe::Response.from_dynamic!(res)
+        client.datascan(query, page: page)
       end
 
       #
@@ -72,33 +74,6 @@ module Mihari
         end
         responses
       end
-
-      #
-      # Build an artifact from an Onyphe search API result
-      #
-      # @param [Structs::Onyphe::Result] result
-      #
-      # @return [Artifact]
-      #
-      def build_artifact(result)
-        as = AutonomousSystem.new(asn: normalize_asn(result.asn))
-
-        geolocation = nil
-        unless result.country_code.nil?
-          geolocation = Geolocation.new(
-            country: NormalizeCountry(result.country_code, to: :short),
-            country_code: result.country_code
-          )
-        end
-
-        Artifact.new(
-          data: result.ip,
-          source: source,
-          metadata: result.metadata,
-          autonomous_system: as,
-          geolocation: geolocation
-        )
-      end
     end
   end
 end

data/lib/mihari/analyzers/otx.rb

@@ -13,6 +13,9 @@ module Mihari
       # @return [String, nil]
       attr_reader :api_key
 
+      # @return [String]
+      attr_reader :query
+
       def initialize(*args, **kwargs)
         super
 
@@ -23,7 +26,14 @@ module Mihari
       end
 
       def artifacts
-        search || []
+        case type
+        when "domain"
+          domain_search
+        when "ip"
+          ip_search
+        else
+          raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
+        end
       end
 
       private
@@ -45,22 +55,6 @@ module Mihari
         %w[ip domain].include? type
       end
 
-      #
-      # IP/domain search
-      #
-      # @return [Array<String>]
-      #
-      def search
-        case type
-        when "domain"
-          domain_search
-        when "ip"
-          ip_search
-        else
-          raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
-        end
-      end
-
       #
       # Domain search
       #

data/lib/mihari/analyzers/passivetotal.rb

@@ -16,6 +16,9 @@ module Mihari
       # @return [String, nil]
       attr_reader :api_key
 
+      # @return [String]
+      attr_reader :query
+
       def initialize(*args, **kwargs)
         super(*args, **kwargs)
 
@@ -27,7 +30,16 @@ module Mihari
       end
 
       def artifacts
-        search || []
+        case type
+        when "domain", "ip"
+          passive_dns_search
+        when "mail"
+          reverse_whois_search
+        when "hash"
+          ssl_search
+        else
+          raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
+        end
       end
 
       def configured?
@@ -53,24 +65,6 @@ module Mihari
         %w[ip domain mail hash].include? type
       end
 
-      #
-      # Passive DNS/SSL, reverse whois search
-      #
-      # @return [Array<String>]
-      #
-      def search
-        case type
-        when "domain", "ip"
-          passive_dns_search
-        when "mail"
-          reverse_whois_search
-        when "hash"
-          ssl_search
-        else
-          raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
-        end
-      end
-
       #
       # Passive DNS search
       #

data/lib/mihari/analyzers/pulsedive.rb

@@ -13,6 +13,9 @@ module Mihari
       # @return [String, nil]
       attr_reader :api_key
 
+      # @return [Integer]
+      attr_reader :query
+
       def initialize(*args, **kwargs)
         super
 
@@ -55,7 +58,6 @@ module Mihari
 
         indicator = client.get_indicator(query)
         iid = indicator["iid"]
-
         properties = client.get_properties(iid)
         (properties["dns"] || []).filter_map do |property|
           if %w[A PTR].include?(property["name"])

data/lib/mihari/analyzers/rule.rb

@@ -7,7 +7,6 @@ module Mihari
       "censys" => Censys,
       "circl" => CIRCL,
       "crtsh" => Crtsh,
-      "dnpedia" => DNPedia,
       "dnstwister" => DNSTwister,
       "feed" => Feed,
       "greynoise" => GreyNoise,