mihari 4.6.1 → 4.7.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 491cb9d0477c4101251c1f3e5705328008a196920efb41c239048247937f21f4
-  data.tar.gz: 14bcfd11b8871a759e70f6658cf2773dadc6afe7ffde44dbbffaef99961b8e00
+  metadata.gz: 7eeecbacefc194f5d2e9356fe8f17884c358b45bb3b7ccbd83a47aa3567ae1f6
+  data.tar.gz: d99abc5ef4368bf1a23a48cf0f15a64a33d3539313a624d489887e86a8cc564c
 SHA512:
-  metadata.gz: d833915545629ade609ec994e6139387fd66247fa1c48fdbe4e3eab01679fb42820e3b07cb95fdbb8cff092753394f1dfa9aaa4459259837c2c0d7d5f2f855ed
-  data.tar.gz: dcbb112faa805b27157ec72d0f3544031c7e7789e855eb082a69a76216d0fcd644013ade95aea28ae138ebfb3bc8eb90d8fb922c83999d292c12b78d15acfe5c
+  metadata.gz: e0347f1c03a949d7e3c85cb9a615247c41bea2a0ac8288f0e3c7a8f9bd5d93ad8f952c99f3b595bf19997fe10e32490898b58a858f63f402a6082344f68cdb87
+  data.tar.gz: a507cfb86593c602ba46ef59f279a17ed1a34e433c8e34898a872048306733d26fb1a6546f1ae96605d524ccd80c3c3a0487ab4b4ad66219c0e3a63215ce8c6e

data/lib/mihari/analyzers/clients/otx.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Analyzers
+    module Clients
+      class OTX
+        attr_reader :api_key
+
+        def initialize(api_key)
+          @api_key = api_key
+        end
+
+        def query_by_ip(ip)
+          get "https://otx.alienvault.com/api/v1/indicators/IPv4/#{ip}/passive_dns"
+        end
+
+        def query_by_domain(domain)
+          get "https://otx.alienvault.com/api/v1/indicators/domain/#{domain}/passive_dns"
+        end
+
+        private
+
+        def headers
+          { "x-otx-api-key": api_key }
+        end
+
+        def get(url)
+          res = HTTP.get(url, headers: headers)
+          JSON.parse(res.body.to_s)
+        rescue HTTPError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/mihari/analyzers/otx.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require "otx_ruby"
+require "mihari/analyzers/clients/otx"
 
 module Mihari
   module Analyzers
@@ -34,12 +34,8 @@ module Mihari
         %w[otx_api_key]
       end
 
-      def domain_client
-        @domain_client ||= ::OTX::Domain.new(api_key)
-      end
-
-      def ip_client
-        @ip_client ||= ::OTX::IP.new(api_key)
+      def client
+        @client ||= Mihari::Analyzers::Clients::OTX.new(api_key)
       end
 
       #
@@ -73,9 +69,15 @@ module Mihari
       # @return [Array<String>]
       #
       def domain_search
-        records = domain_client.get_passive_dns(query)
+        res = client.query_by_domain(query)
+        return [] if res.nil?
+
+        records = res["passive_dns"] || []
         records.filter_map do |record|
-          record.address if record.record_type == "A"
+          record_type = record["record_type"]
+          address = record["address"]
+
+          address if record_type == "A"
         end.uniq
       end
 
@@ -85,9 +87,15 @@ module Mihari
       # @return [Array<String>]
       #
       def ip_search
-        records = ip_client.get_passive_dns(query)
+        res = client.query_by_ip(query)
+        return [] if res.nil?
+
+        records = res["passive_dns"] || []
         records.filter_map do |record|
-          record.hostname if record.record_type == "A"
+          record_type = record["record_type"]
+          hostname = record["hostname"]
+
+          hostname if record_type == "A"
         end.uniq
       end
     end

data/lib/mihari/analyzers/rule.rb

@@ -39,7 +39,6 @@ module Mihari
 
     class Rule < Base
       include Mixins::DisallowedDataValue
-      include Mixins::Rule
 
       option :title
       option :description
@@ -51,6 +50,7 @@ module Mihari
       option :disallowed_data_values, default: proc { [] }
 
       option :emitters, optional: true
+      option :enrichers, optional: true
 
       attr_reader :source
 
@@ -60,6 +60,7 @@ module Mihari
         @source = id
 
         @emitters = emitters || DEFAULT_EMITTERS
+        @enrichers = enrichers || DEFAULT_ENRICHERS
 
         validate_analyzer_configurations
       end
@@ -112,6 +113,21 @@ module Mihari
         end
       end
 
+      #
+      # Enriched artifacts
+      #
+      # @return [Array<Mihari::Artifact>]
+      #
+      def enriched_artifacts
+        @enriched_artifacts ||= Parallel.map(unique_artifacts) do |artifact|
+          enrichers.each do |enricher|
+            artifact.enrich_by_enricher(enricher[:enricher])
+          end
+
+          artifact
+        end
+      end
+
       #
       # Normalized disallowed data values
       #

data/lib/mihari/commands/init.rb

@@ -3,8 +3,6 @@
 module Mihari
   module Commands
     module Initialization
-      include Mixins::Rule
-
       def self.included(thor)
         thor.class_eval do
           desc "rule", "Create a rule file"
@@ -21,6 +19,31 @@ module Mihari
 
             Mihari.logger.info "The rule file is initialized as #{filename}."
           end
+
+          no_commands do
+            #
+            # Returns a template for rule
+            #
+            # @return [String] A template for rule
+            #
+            def rule_template
+              rule = Structs::Rule.from_path_or_id File.expand_path("../templates/rule.yml.erb", __dir__)
+              rule.yaml
+            end
+
+            #
+            # Create (blank) rule file
+            #
+            # @param [String] filename
+            # @param [Dry::Files] files
+            # @param [String] template
+            #
+            # @return [nil]
+            #
+            def initialize_rule_yaml(filename, files = Dry::Files.new, template: rule_template)
+              files.write(filename, template)
+            end
+          end
         end
       end
     end

data/lib/mihari/commands/search.rb

@@ -4,7 +4,6 @@ module Mihari
   module Commands
     module Search
       include Mixins::Database
-      include Mixins::Rule
       include Mixins::ErrorNotification
 
       def self.included(thor)
@@ -12,14 +11,10 @@ module Mihari
           desc "search [RULE]", "Search by a rule"
           method_option :yes, type: :boolean, aliases: "-y", desc: "yes to overwrite the rule in the database"
           def search_by_rule(path_or_id)
-            rule = load_rule(path_or_id)
+            rule = Structs::Rule.from_path_or_id path_or_id
 
             # validate
-            begin
-              validate_rule! rule
-            rescue RuleValidationError => e
-              raise e
-            end
+            rule.validate!
 
             # check update
             id = rule.id

data/lib/mihari/commands/validator.rb

@@ -3,16 +3,21 @@
 module Mihari
   module Commands
     module Validator
-      include Mixins::Rule
-
       def self.included(thor)
         thor.class_eval do
-          desc "rule [PATH]", "Validate format of a rule file"
+          desc "rule [PATH]", "Validate rule file format"
+          #
+          # Validate format of a rule
+          #
+          # @param [String] path
+          #
+          # @return [nil]
+          #
           def rule(path)
-            rule = load_rule(path)
+            rule = Structs::Rule.from_path_or_id(path)
 
             begin
-              validate_rule! rule
+              rule.validate!
               Mihari.logger.info "Valid format. The input is parsed as the following:\n#{rule.data.to_yaml}"
             rescue RuleValidationError
               nil

data/lib/mihari/constants.rb

@@ -4,4 +4,6 @@ module Mihari
   ALLOWED_DATA_TYPES = ["hash", "ip", "domain", "url", "mail"].freeze
 
   DEFAULT_EMITTERS = ["database", "misp", "slack", "the_hive", "webhook"].map { |name| { emitter: name } }.freeze
+
+  DEFAULT_ENRICHERS = ["whois", "ipinfo", "shodan", "google_public_dns"].map { |name| { enricher: name } }.freeze
 end

data/lib/mihari/enrichers/google_public_dns.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require "net/https"
+
+module Mihari
+  module Enrichers
+    class GooglePublicDNS < Base
+      # @return [Boolean]
+      def valid?
+        true
+      end
+
+      class << self
+        #
+        # Query Google Public DNS
+        #
+        # @param [String] name
+        # @param [String] resource_type
+        #
+        # @return [Mihari::Structs::Shodan::GooglePublicDNS::Response, nil]
+        #
+        def query(name, resource_type)
+          url = "https://dns.google/resolve"
+          params = { name: name, type: resource_type }
+          res = HTTP.get(url, params: params)
+
+          data = JSON.parse(res.body.to_s)
+
+          Structs::GooglePublicDNS::Response.from_dynamic! data
+        rescue HTTPError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/mihari/enrichers/whois.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+require "whois-parser"
+
+module Mihari
+  module Enrichers
+    class Whois < Base
+      @memo = {}
+
+      # @return [Boolean]
+      def valid?
+        true
+      end
+
+      class << self
+        #
+        # Query IAIA Whois API
+        #
+        # @param [String] name
+        #
+        # @return [Mihari::WhoisRecord, nil]
+        #
+        def query(domain)
+          domain = PublicSuffix.domain(domain)
+
+          # check memo
+          if @memo.key?(domain)
+            whois_record = @memo[domain]
+            # return clone of the record
+            return whois_record.dup
+          end
+
+          record = ::Whois.whois(domain)
+          parser = record.parser
+
+          return nil if parser.available?
+
+          whois_record = WhoisRecord.new(
+            domain: domain,
+            created_on: get_created_on(parser),
+            updated_on: get_updated_on(parser),
+            expires_on: get_expires_on(parser),
+            registrar: get_registrar(parser),
+            contacts: get_contacts(parser)
+          )
+          # set memo
+          @memo[domain] = whois_record
+          whois_record
+        rescue ::Whois::Error, ::Whois::ParserError, Timeout::Error
+          nil
+        end
+
+        def reset_cache
+          @memo = {}
+        end
+
+        private
+
+        #
+        # Get created_on
+        #
+        # @param [::Whois::Parser:] parser
+        #
+        # @return [Date, nil]
+        #
+        def get_created_on(parser)
+          parser.created_on
+        rescue ::Whois::AttributeNotImplemented
+          nil
+        end
+
+        #
+        # Get updated_on
+        #
+        # @param [::Whois::Parser:] parser
+        #
+        # @return [Date, nil]
+        #
+        def get_updated_on(parser)
+          parser.updated_on
+        rescue ::Whois::AttributeNotImplemented
+          nil
+        end
+
+        #
+        # Get expires_on
+        #
+        # @param [::Whois::Parser:] parser
+        #
+        # @return [Date, nil]
+        #
+        def get_expires_on(parser)
+          parser.expires_on
+        rescue ::Whois::AttributeNotImplemented
+          nil
+        end
+
+        #
+        # Get registrar
+        #
+        # @param [::Whois::Parser:] parser
+        #
+        # @return [Hash, nil]
+        #
+        def get_registrar(parser)
+          parser.registrar&.to_h
+        rescue ::Whois::AttributeNotImplemented
+          nil
+        end
+
+        #
+        # Get contacts
+        #
+        # @param [::Whois::Parser:] parser
+        #
+        # @return [Array[Hash], nil]
+        #
+        def get_contacts(parser)
+          parser.contacts.map(&:to_h)
+        rescue ::Whois::AttributeNotImplemented
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/mihari/errors.rb

@@ -15,6 +15,8 @@ module Mihari
 
   class RuleValidationError < Error; end
 
+  class YAMLSyntaxError < Error; end
+
   class ConfigurationError < Error; end
 
   class HTTPError < Error; end

data/lib/mihari/http.rb

@@ -44,8 +44,8 @@ module Mihari
     end
 
     class << self
-      def get(uri, headers: {}, payload: {})
-        client = new(uri, headers: headers, payload: payload)
+      def get(uri, headers: {}, params: {})
+        client = new(uri, headers: headers, payload: params)
         client.get
       end
 

data/lib/mihari/models/alert.rb

@@ -12,7 +12,7 @@ module Mihari
       #
       # Search alerts
       #
-      # @param [Structs::Alert::SearchFilterWithPagination] filter
+      # @param [Structs::Filters::Alert::SearchFilterWithPagination] filter
       #
       # @return [Array<Alert>]
       #
@@ -58,6 +58,11 @@ module Mihari
 
       private
 
+      #
+      # @param [Structs::Filters::Alert::SearchFilter] filter
+      #
+      # @return [Mihari::Alert]
+      #
       def build_relation(filter)
         artifact_ids = []
         artifact = Artifact.includes(:autonomous_system, :dns_records, :reverse_dns_names)

data/lib/mihari/models/artifact.rb

@@ -135,6 +135,36 @@ module Mihari
       enrich_cpes
     end
 
+    ENRICH_METHODS_BY_ENRICHER = {
+      whois: [
+        :enrich_whois
+      ],
+      ipinfo: [
+        :enrich_autonomous_system,
+        :enrich_geolocation
+      ],
+      shodan: [
+        :enrich_ports,
+        :enrich_cpes,
+        :enrich_reverse_dns
+      ],
+      google_public_dns: [
+        :enrich_dns
+      ]
+    }.freeze
+
+    #
+    # Enrich by name of enricher
+    #
+    # @param [String] enricher
+    #
+    def enrich_by_enricher(enricher)
+      methods = ENRICH_METHODS_BY_ENRICHER[enricher.downcase.to_sym] || []
+      methods.each do |method|
+        send(method) if respond_to?(method)
+      end
+    end
+
     private
 
     def normalize_as_domain(url_or_domain)

data/lib/mihari/models/dns.rb

@@ -13,14 +13,7 @@ module Mihari
       # @return [Array<Mihari::DnsRecord>]
       #
       def build_by_domain(domain)
-        resource_types = [
-          Resolv::DNS::Resource::IN::A,
-          Resolv::DNS::Resource::IN::AAAA,
-          Resolv::DNS::Resource::IN::CNAME,
-          Resolv::DNS::Resource::IN::TXT,
-          Resolv::DNS::Resource::IN::NS
-        ]
-
+        resource_types = %w[A AAAA CNAME TXT NS]
         resource_types.map do |resource_type|
           get_values domain, resource_type
         rescue Resolv::ResolvError
@@ -31,20 +24,11 @@ module Mihari
       private
 
       def get_values(domain, resource_type)
-        resources = Resolv::DNS.new.getresources(domain, resource_type)
-        resource_name = resource_type.to_s.split("::").last
+        response = Enrichers::GooglePublicDNS.query(domain, resource_type)
+        answers = response.answers || []
 
-        resources.filter_map do |resource|
-          # A, AAAA
-          if resource.respond_to?(:address)
-            new(resource: resource_name, value: resource.address.to_s)
-          # CNAME, NS
-          elsif resource.respond_to?(:name)
-            new(resource: resource_name, value: resource.name.to_s)
-          # TXT
-          elsif resource.respond_to?(:data)
-            new(resource: resource_name, value: resource.data.to_s)
-          end
+        answers.filter_map do |answer|
+          new(resource: answer.resource_type, value: answer.data)
         end
       end
     end

data/lib/mihari/models/geolocation.rb

@@ -17,11 +17,9 @@ module Mihari
       def build_by_ip(ip)
         res = Enrichers::IPInfo.query(ip)
 
-        unless res.nil?
-          return new(country: NormalizeCountry(res.country_code, to: :short), country_code: res.country_code)
-        end
+        return nil if res&.country_code.nil?
 
-        nil
+        new(country: NormalizeCountry(res.country_code, to: :short), country_code: res.country_code)
       end
     end
   end

data/lib/mihari/models/port.rb

@@ -14,7 +14,7 @@ module Mihari
       #
       def build_by_ip(ip)
         res = Enrichers::Shodan.query(ip)
-        return if res.nil?
+        return [] if res.nil?
 
         res.ports.map { |port| new(port: port) }
       end

data/lib/mihari/models/rule.rb

@@ -28,7 +28,7 @@ module Mihari
       #
       # Search rules
       #
-      # @param [Structs::Rule::SearchFilterWithPagination] filter
+      # @param [Structs::Filters::Rule::SearchFilterWithPagination] filter
       #
       # @return [Array<Rule>]
       #
@@ -51,7 +51,7 @@ module Mihari
       #
       # Count alerts
       #
-      # @param [Structs::Rule::SearchFilterWithPagination] filter
+      # @param [Structs::Filters::Rule::SearchFilterWithPagination] filter
       #
       # @return [Integer]
       #
@@ -62,6 +62,11 @@ module Mihari
 
       private
 
+      #
+      # @param [Structs::Filters::Rule::SearchFilter] filter
+      #
+      # @return [Mihari::Rule]
+      #
       def build_relation(filter)
         relation = self
         relation = relation.includes(alerts: :tags)