mihari 3.6.1 → 3.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 14d0c74e85fbf6ef624afefe7e948595586d6c00fa8bc32f211e60caee581fc3
-  data.tar.gz: 9d5fde6d69f664efac0d6c56e6a0ba60adcad0edcfe45c69f285ffcaba8d11f0
+  metadata.gz: aac91d43689cb53dc0570bfed3cec57a07cbe88de0716530f2ea8bfac8f8d39d
+  data.tar.gz: 0f59bdc53cfa75e56884dd3497fa0492d3a41a3b7540cbdab1345ec5b301c69c
 SHA512:
-  metadata.gz: 30597743e91f124388fbdf426199d97a00f92db9e525fe5725643d529d319ca670d1e9aa6eccff86c0844802157ca3d5c7db9b9afd925d6f0ac4d8b881c44949
-  data.tar.gz: 9d199a3c2f6c7794214730de7c8db812e939c5ddd11ab06d1c6f28c3b8764b2881958b0684adb59e8516d0ac4b6a1dc66b19c5a593efaac9695cb6e317ce8105
+  metadata.gz: 30aef30fb14c7c1a50e75162141d1266b1ec5b847f6329935a221a90f59d37a2bed97c6a8aa4371962ab85b18c5ff23cf0417f08f9dc3320c737721ac1a07602
+  data.tar.gz: 1029878ec85cbdbe0a2c800b6b68cce99d45510818dde6b1d84a826022379cbe21fc57f490e3a77521136dbc22e8112b804d891892ea0a351e0d1db935b28b4b

data/README.md

@@ -46,7 +46,7 @@ Mihari supports the following services by default.
 - [Shodan](https://shodan.io)
 - [Spyse](https://spyse.com)
 - [urlscan.io](https://urlscan.io)
-- [VirusTotal](http://virustotal.com)
+- [VirusTotal](http://virustotal.com) & [VirusTotal Intelligence](https://www.virustotal.com/gui/intelligence-overview)
 - [ZoomEye](https://zoomeye.org)
 
 ## Docs
@@ -64,5 +64,3 @@ The gem is available as open source under the terms of the [MIT License](https:/
 ## Acknowledgement
 
 Mihari is proudly supported by [Tines.io](https://tines.io?utm_source=github&utm_medium=sponsorship&utm_campaign=ninoseki), The SOAR Platform for Enterprise Security Teams.
-
-$ bundle exec rbs -rpathname  --repo=gem_rbs/gems -ractivesupport -ractionpack -ractivejob -ractivemodel -ractionview -ractiverecord -rrailties -I sig validate

data/lib/mihari/analyzers/base.rb

@@ -8,6 +8,7 @@ module Mihari
     class Base
       extend Dry::Initializer
 
+      include Mixins::AutonomousSystem
       include Mixins::Configurable
       include Mixins::Retriable
 
@@ -111,9 +112,7 @@ module Mihari
       #
       def enriched_artifacts
         @enriched_artifacts ||= unique_artifacts.map do |artifact|
-          artifact.enrich_whois
-          artifact.enrich_dns
-          artifact.enrich_reverse_dns
+          artifact.enrich_all
           artifact
         end
       end
@@ -141,20 +140,6 @@ module Mihari
           emitter.valid? ? emitter : nil
         end.compact
       end
-
-      #
-      # Normalize ASN value
-      #
-      # @param [String, Integer] asn
-      #
-      # @return [Integer]
-      #
-      def normalize_asn(asn)
-        return asn if asn.is_a?(Integer)
-        return asn.to_i unless asn.start_with?("AS")
-
-        asn.delete_prefix("AS").to_i
-      end
     end
   end
 end

data/lib/mihari/analyzers/rule.rb

@@ -42,6 +42,7 @@ module Mihari
         "spyse" => Spyse,
         "urlscan" => Urlscan,
         "virustotal" => VirusTotal,
+        "virustotal_intelligence" => VirusTotalIntelligence,
         "zoomeye" => ZoomEye
       }.freeze
 

data/lib/mihari/analyzers/virustotal_intelligence.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require "virustotal"
+
+module Mihari
+  module Analyzers
+    class VirusTotalIntelligence < Base
+      param :query
+      option :title, default: proc { "VirusTotal Intelligence search" }
+      option :description, default: proc { "query = #{query}" }
+      option :tags, default: proc { [] }
+
+      def initialize(*args, **kwargs)
+        super
+
+        @query = query
+      end
+
+      def artifacts
+        responses = search_witgh_cursor
+        responses.map do |response|
+          response.data.map(&:value)
+        end.flatten.compact.uniq
+      end
+
+      private
+
+      def configuration_keys
+        %w[virustotal_api_key]
+      end
+
+      #
+      # VT API
+      #
+      # @return [::VirusTotal::API]
+      #
+      def api
+        @api = ::VirusTotal::API.new(key: Mihari.config.virustotal_api_key)
+      end
+
+      #
+      # Search with cursor
+      #
+      # @return [Array<Structs::VirusTotalIntelligence::Response>]
+      #
+      def search_witgh_cursor
+        cursor = nil
+        responses = []
+
+        loop do
+          response = Structs::VirusTotalIntelligence::Response.from_dynamic!(api.intelligence.search(query, cursor: cursor))
+          responses << response
+
+          break if response.meta.cursor.nil?
+
+          cursor = response.meta.cursor
+        end
+
+        responses
+      end
+    end
+  end
+end

data/lib/mihari/cli/analyzer.rb

@@ -14,6 +14,7 @@ require "mihari/commands/securitytrails"
 require "mihari/commands/shodan"
 require "mihari/commands/spyse"
 require "mihari/commands/urlscan"
+require "mihari/commands/virustotal_intelligence"
 require "mihari/commands/virustotal"
 require "mihari/commands/zoomeye"
 
@@ -42,6 +43,7 @@ module Mihari
       include Mihari::Commands::Spyse
       include Mihari::Commands::Urlscan
       include Mihari::Commands::VirusTotal
+      include Mihari::Commands::VirusTotalIntelligence
       include Mihari::Commands::ZoomEye
     end
   end

data/lib/mihari/commands/passivetotal.rb

@@ -14,6 +14,7 @@ module Mihari
               run_analyzer Analyzers::PassiveTotal, query: indicator, options: options
             end
           end
+          map "pt" => :passivetotal
         end
       end
     end

data/lib/mihari/commands/virustotal.rb

@@ -14,6 +14,7 @@ module Mihari
               run_analyzer Analyzers::VirusTotal, query: indiactor, options: options
             end
           end
+          map "vt" => :virustotal
         end
       end
     end

data/lib/mihari/commands/virustotal_intelligence.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Commands
+    module VirusTotalIntelligence
+      def self.included(thor)
+        thor.class_eval do
+          desc "virustotal_intelligence [QUERY]", "VirusTotal Intelligence search"
+          method_option :title, type: :string, desc: "title"
+          method_option :description, type: :string, desc: "description"
+          method_option :tags, type: :array, desc: "tags"
+          def virustotal_intelligence(query)
+            with_error_handling do
+              run_analyzer Analyzers::VirusTotalIntelligence, query: query, options: options
+            end
+          end
+          map "vt_intel" => :virustotal_intelligence
+        end
+      end
+    end
+  end
+end

data/lib/mihari/database.rb

@@ -74,6 +74,17 @@ class EnrichmentsSchema < ActiveRecord::Migration[6.1]
   end
 end
 
+class EnrichmentCreatedAtSchema < ActiveRecord::Migration[6.1]
+  def change
+    # Add created_at column because now it is able to enrich an atrifact after the creation
+    add_column :autonomous_systems, :created_at, :datetime, if_not_exists: true
+    add_column :geolocations, :created_at, :datetime, if_not_exists: true
+    add_column :whois_records, :created_at, :datetime, if_not_exists: true
+    add_column :dns_records, :created_at, :datetime, if_not_exists: true
+    add_column :reverse_dns_names, :created_at, :datetime, if_not_exists: true
+  end
+end
+
 def adapter
   return "postgresql" if Mihari.config.database.start_with?("postgresql://", "postgres://")
   return "mysql2" if Mihari.config.database.start_with?("mysql2://")
@@ -101,6 +112,7 @@ module Mihari
         InitialSchema.migrate(:up)
         AddeSourceToArtifactSchema.migrate(:up)
         EnrichmentsSchema.migrate(:up)
+        EnrichmentCreatedAtSchema.migrate(:up)
       rescue StandardError
         # Do nothing
       end
@@ -116,6 +128,7 @@ module Mihari
         InitialSchema.migrate(:down)
         AddeSourceToArtifactSchema.migrate(:down)
         EnrichmentsSchema.migrate(:down)
+        EnrichmentCreatedAtSchema.migrate(:down)
       end
     end
   end

data/lib/mihari/enrichers/base.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Enrichers
+    class Base
+      include Mixins::Configurable
+
+      def self.inherited(child)
+        Mihari.enrichers << child
+      end
+
+      # @return [Boolean]
+      def valid?
+        raise NotImplementedError, "You must implement #{self.class}##{__method__}"
+      end
+    end
+  end
+end

data/lib/mihari/enrichers/ipinfo.rb

@@ -0,0 +1,49 @@
+require "http"
+require "json"
+require "memist"
+
+module Mihari
+  module Enrichers
+    class IPInfo < Base
+      # @return [Boolean]
+      def valid?
+        Mihari.config.ipinfo_api_key.nil?
+      end
+
+      private
+
+      def configuration_keys
+        %w[ipinfo_api_key]
+      end
+
+      class << self
+        include Memist::Memoizable
+
+        #
+        # Query IPInfo
+        #
+        # @param [String] ip
+        #
+        # @return [Mihari::Structs::IPInfo::Response, nil]
+        #
+        def query(ip)
+          headers = {}
+          token = Mihari.config.ipinfo_api_key
+          unless token.nil?
+            headers[:authorization] = "Bearer #{token}"
+          end
+
+          begin
+            res = HTTP.headers(headers).get("https://ipinfo.io/#{ip}/json")
+            data = JSON.parse(res.body.to_s)
+
+            Structs::IPInfo::Response.from_dynamic! data
+          rescue HTTP::Error
+            nil
+          end
+        end
+        memoize :query
+      end
+    end
+  end
+end

data/lib/mihari/mixins/autonomous_system.rb

@@ -0,0 +1,19 @@
+module Mihari
+  module Mixins
+    module AutonomousSystem
+      #
+      # Normalize ASN value
+      #
+      # @param [String, Integer] asn
+      #
+      # @return [Integer]
+      #
+      def normalize_asn(asn)
+        return asn if asn.is_a?(Integer)
+        return asn.to_i unless asn.start_with?("AS")
+
+        asn.delete_prefix("AS").to_i
+      end
+    end
+  end
+end

data/lib/mihari/models/artifact.rb

@@ -16,6 +16,8 @@ end
 
 module Mihari
   class Artifact < ActiveRecord::Base
+    belongs_to :alert
+
     has_one :autonomous_system, dependent: :destroy
     has_one :geolocation, dependent: :destroy
     has_one :whois_record, dependent: :destroy
@@ -80,6 +82,35 @@ module Mihari
       self.reverse_dns_names = ReverseDnsName.build_by_ip(data)
     end
 
+    #
+    # Enrich(add) geolocation
+    #
+    def enrich_geolocation
+      return unless can_enrich_geolocation?
+
+      self.geolocation = Geolocation.build_by_ip(data)
+    end
+
+    #
+    # Enrich(add) geolocation
+    #
+    def enrich_autonomous_system
+      return unless can_enrich_autonomous_system?
+
+      self.autonomous_system = AutonomousSystem.build_by_ip(data)
+    end
+
+    #
+    # Enrich all the enrichable relationships of the artifact
+    #
+    def enrich_all
+      enrich_autonomous_system
+      enrich_dns
+      enrich_geolocation
+      enrich_reverse_dns
+      enrich_whois
+    end
+
     private
 
     def normalize_as_domain(url_or_domain)
@@ -89,15 +120,23 @@ module Mihari
     end
 
     def can_enrich_whois?
-      %w[domain url].include? data_type
+      %w[domain url].include?(data_type) && whois_record.nil?
     end
 
     def can_enrich_dns?
-      %w[domain url].include? data_type
+      %w[domain url].include?(data_type) && dns_records.empty?
     end
 
     def can_enrich_revese_dns?
-      data_type == "ip"
+      data_type == "ip" && reverse_dns_names.empty?
+    end
+
+    def can_enrich_geolocation?
+      data_type == "ip" && geolocation.nil?
+    end
+
+    def can_enrich_autonomous_system?
+      data_type == "ip" && autonomous_system.nil?
     end
   end
 end

data/lib/mihari/models/autonomous_system.rb

@@ -4,6 +4,23 @@ require "active_record"
 
 module Mihari
   class AutonomousSystem < ActiveRecord::Base
-    has_one :artifact, dependent: :destroy
+    belongs_to :artifact
+
+    class << self
+      #
+      # Build AS
+      #
+      # @param [String] ip
+      #
+      # @return [Mihari::AutonomousSystem, nil]
+      #
+      def build_by_ip(ip)
+        res = Enrichers::IPInfo.query(ip)
+
+        return nil if res.nil? || res.asn.nil?
+
+        new(asn: res.asn)
+      end
+    end
   end
 end

data/lib/mihari/models/dns.rb

@@ -5,6 +5,8 @@ require "resolv"
 
 module Mihari
   class DnsRecord < ActiveRecord::Base
+    belongs_to :artifact
+
     class << self
       #
       # Build DNS records

data/lib/mihari/models/geolocation.rb

@@ -1,9 +1,29 @@
 # frozen_string_literal: true
 
 require "active_record"
+require "normalize_country"
 
 module Mihari
   class Geolocation < ActiveRecord::Base
-    has_one :artifact, dependent: :destroy
+    belongs_to :artifact
+
+    class << self
+      #
+      # Build Geolocation
+      #
+      # @param [String] ip
+      #
+      # @return [Mihari::Geolocation, nil]
+      #
+      def build_by_ip(ip)
+        res = Enrichers::IPInfo.query(ip)
+
+        unless res.nil?
+          return new(country: NormalizeCountry(res.country_code, to: :short), country_code: res.country_code)
+        end
+
+        nil
+      end
+    end
   end
 end

data/lib/mihari/models/reverse_dns.rb

@@ -5,6 +5,8 @@ require "resolv"
 
 module Mihari
   class ReverseDnsName < ActiveRecord::Base
+    belongs_to :artifact
+
     class << self
       #
       # Build reverse DNS names

data/lib/mihari/models/whois.rb

@@ -6,7 +6,7 @@ require "public_suffix"
 
 module Mihari
   class WhoisRecord < ActiveRecord::Base
-    has_one :artifact, dependent: :destroy
+    belongs_to :artifact
 
     @memo = {}
 

data/lib/mihari/status.rb

@@ -18,7 +18,7 @@ module Mihari
     # @return [Array<Hash>]
     #
     def statuses
-      (Mihari.analyzers + Mihari.emitters).map do |klass|
+      (Mihari.analyzers + Mihari.emitters + Mihari.enrichers).map do |klass|
         name = klass.to_s.split("::").last.to_s
 
         [name, build_status(klass)]
@@ -36,11 +36,16 @@ module Mihari
       return nil if klass == Mihari::Analyzers::Rule
 
       is_analyzer = klass.ancestors.include?(Mihari::Analyzers::Base)
+      is_emitter = klass.ancestors.include?(Mihari::Emitters::Base)
+      is_enricher = klass.ancestors.include?(Mihari::Enrichers::Base)
 
       instance = is_analyzer ? klass.new("dummy") : klass.new
       is_configured = instance.configured?
       values = instance.configuration_values
-      type = is_analyzer ? "Analyzer" : "Emitter"
+
+      type = "Analyzer"
+      type = "Emitter" if is_emitter
+      type = "Enricher" if is_enricher
 
       values ? { is_configured: is_configured, values: values, type: type } : nil
     rescue ArgumentError => _e

data/lib/mihari/structs/ipinfo.rb

@@ -0,0 +1,39 @@
+require "json"
+require "dry/struct"
+
+module Mihari
+  module Structs
+    module IPInfo
+      class Response < Dry::Struct
+        attribute :ip, Types::String
+        attribute :hostname, Types::String.optional
+        attribute :loc, Types::String
+        attribute :country_code, Types::String
+        attribute :asn, Types::Integer.optional
+
+        class << self
+          include Mixins::AutonomousSystem
+
+          def from_dynamic!(d)
+            d = Types::Hash[d]
+
+            asn = nil
+            org = d["org"]
+            unless org.nil?
+              asn = org.split.first
+              asn = normalize_asn(asn)
+            end
+
+            new(
+              ip: d.fetch("ip"),
+              loc: d.fetch("loc"),
+              hostname: d["hostname"],
+              country_code: d.fetch("country"),
+              asn: asn
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mihari/structs/virustotal_intelligence.rb

@@ -0,0 +1,75 @@
+require "json"
+require "dry/struct"
+
+module Mihari
+  module Structs
+    module VirusTotalIntelligence
+      class ContextAttributes < Dry::Struct
+        attribute :url, Types.Array(Types::String).optional
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            url: d["url"]
+          )
+        end
+      end
+
+      class Datum < Dry::Struct
+        attribute :type, Types::String
+        attribute :id, Types::String
+        attribute :context_attributes, ContextAttributes.optional
+
+        def value
+          case type
+          when "file"
+            id
+          when "url"
+            (context_attributes.url || []).first
+          when "domain"
+            id
+          when "ip_address"
+            id
+          end
+        end
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+
+          context_attributes = nil
+          context_attributes = ContextAttributes.from_dynamic!(d.fetch("context_attributes")) if d.key?("context_attributes")
+
+          new(
+            type: d.fetch("type"),
+            id: d.fetch("id"),
+            context_attributes: context_attributes
+          )
+        end
+      end
+
+      class Meta < Dry::Struct
+        attribute :cursor, Types::String.optional
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            cursor: d["cursor"]
+          )
+        end
+      end
+
+      class Response < Dry::Struct
+        attribute :meta, Meta
+        attribute :data, Types.Array(Datum)
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            meta: Meta.from_dynamic!(d.fetch("meta")),
+            data: d.fetch("data").map { |x| Datum.from_dynamic!(x) }
+          )
+        end
+      end
+    end
+  end
+end

data/lib/mihari/types.rb

@@ -13,9 +13,19 @@ module Mihari
     DataTypes = Types::String.enum(*ALLOWED_DATA_TYPES)
 
     AnalyzerTypes = Types::String.enum(
-      "binaryedge", "censys", "circl", "dnpedia", "dnstwister",
-      "onyphe", "otx", "passivetotal", "pulsedive", "securitytrails",
-      "shodan", "virustotal"
+      "binaryedge",
+      "censys",
+      "circl",
+      "dnpedia",
+      "dnstwister",
+      "onyphe",
+      "otx",
+      "passivetotal",
+      "pulsedive",
+      "securitytrails",
+      "shodan",
+      "virustotal_intelligence",
+      "virustotal"
     )
   end
 end

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "3.6.1"
+  VERSION = "3.8.0"
 end