mihari 3.5.0 → 3.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a2fe0203f89908abbc21df8d595b7167a561c826b933166e531298b83f67e085
-  data.tar.gz: 03a455ebd71f5d3c228041351b6ee8a6a2d89b74a7fc2f0d7609293bc82da7d6
+  metadata.gz: c9c1cbdf0570c25e2d89d7f6fd402b64991dfaebc75cf3cf5422a56504287ae9
+  data.tar.gz: d5b7a0db7b49f245e3c949135011fb674e28a9d3c251e165f827e8f3d90673b1
 SHA512:
-  metadata.gz: aa4a28142eb109d46109d960c1de935ac4632136bc7696b405bfda3d691de2658764e7f796a8e9b1715973e0fc3cc9887997f3ab0dd054e4caa6baf6da40cb8e
-  data.tar.gz: 057c5df7efd59ebf285b4a8d6a17a8857d8fa5df7b0c61f4ddbaf67d3688aaa4c34187af88acfe151a27a21b1d33b5e659347968ba6881e98e3b81114d113eb9
+  metadata.gz: e76a216dedbc1aec17748c37a1b874c2c825fed6f7716ef356a48ddf2861584da299c384737e588a48b67165874f495192bb42a4c20c2f29f4620f8b559d1a83
+  data.tar.gz: 2f3e380b252ba238594ccacd2df8e362a62b9685aafeff34d6506e7301184cf5b38c4244db9498842f4aee9df109d7c43a9ad99cecc3b63616d333a7f5093333

data/config.ru

@@ -1,3 +1,4 @@
+# bundle exec rerun -- rackup config.ru
 require "./lib/mihari"
 
 # set rack env as development

data/lib/mihari/analyzers/base.rb

@@ -51,7 +51,7 @@ module Mihari
       # @return [nil]
       #
       def run
-        set_unique_artifacts
+        set_enriched_artifacts
 
         Parallel.each(valid_emitters) do |emitter|
           run_emitter emitter
@@ -66,7 +66,7 @@ module Mihari
       # @return [nil]
       #
       def run_emitter(emitter)
-        emitter.run(title: title, description: description, artifacts: unique_artifacts, source: source, tags: tags)
+        emitter.run(title: title, description: description, artifacts: enriched_artifacts, source: source, tags: tags)
       rescue StandardError => e
         puts "Emission by #{emitter.class} is failed: #{e}"
       end
@@ -88,7 +88,7 @@ module Mihari
           # No need to set data_type manually
           # It is set automatically in #initialize
           artifact.is_a?(Artifact) ? artifact : Artifact.new(data: artifact, source: source)
-        end.select(&:valid?)
+        end.select(&:valid?).uniq(&:data)
       end
 
       private
@@ -105,12 +105,26 @@ module Mihari
       end
 
       #
-      # Set unique artifacts
+      # Enriched artifacts
+      #
+      # @return [Array<Mihari::Artifact>]
+      #
+      def enriched_artifacts
+        @enriched_artifacts ||= unique_artifacts.map do |artifact|
+          artifact.enrich_whois
+          artifact.enrich_dns
+          artifact.enrich_reverse_dns
+          artifact
+        end
+      end
+
+      #
+      # Set enriched artifacts
       #
       # @return [nil]
       #
-      def set_unique_artifacts
-        retry_on_error { unique_artifacts }
+      def set_enriched_artifacts
+        retry_on_error { enriched_artifacts }
       rescue ArgumentError => _e
         klass = self.class.to_s.split("::").last.to_s
         raise Error, "Please configure #{klass} API settings properly"
@@ -127,6 +141,20 @@ module Mihari
           emitter.valid? ? emitter : nil
         end
       end
+
+      #
+      # Normalize ASN value
+      #
+      # @param [String, Integer] asn
+      #
+      # @return [Integer]
+      #
+      def normalize_asn(asn)
+        return asn if asn.is_a?(Integer)
+        return asn.to_i unless asn.start_with?("AS")
+
+        asn.delete_prefix("AS").to_i
+      end
     end
   end
 end

data/lib/mihari/analyzers/censys.rb

@@ -17,31 +17,59 @@ module Mihari
       private
 
       def search
-        ipv4s = []
+        artifacts = []
 
         cursor = nil
         loop do
           response = api.search(query, cursor: cursor)
-          ipv4s << response_to_ipv4s(response)
+          response = Structs::Censys::Response.from_dynamic!(response)
 
-          links = response.dig("result", "links")
-          cursor = links["next"]
+          artifacts << response_to_artifacts(response)
+
+          cursor = response.result.links.next
           break if cursor == ""
         end
 
-        ipv4s.flatten
+        artifacts.flatten.uniq(&:data)
       end
 
       #
       # Extract IPv4s from Censys search API response
       #
-      # @param [Hash] response
+      # @param [Structs::Censys::Response] response
       #
       # @return [Array<String>]
       #
-      def response_to_ipv4s(response)
-        hits = response.dig("result", "hits") || []
-        hits.map { |hit| hit["ip"] }
+      def response_to_artifacts(response)
+        response.result.hits.map { |hit| build_artifact(hit) }
+      end
+
+      #
+      # Build an artifact from a Shodan search API response
+      #
+      # @param [Structs::Censys::Hit] hit
+      #
+      # @return [Artifact]
+      #
+      def build_artifact(hit)
+        as = AutonomousSystem.new(asn: normalize_asn(hit.autonomous_system.asn))
+
+        # sometimes Censys overlooks country
+        # then set geolocation as nil
+        geolocation = nil
+        unless hit.location.country.nil?
+          geolocation = Geolocation.new(
+            country: hit.location.country,
+            country_code: hit.location.country_code
+          )
+        end
+
+        Artifact.new(
+          data: hit.ip,
+          source: source,
+          autonomous_system: as,
+          geolocation: geolocation
+        )
       end
 
       def configuration_keys

data/lib/mihari/analyzers/onyphe.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "onyphe"
+require "normalize_country"
 
 module Mihari
   module Analyzers
@@ -11,14 +12,13 @@ module Mihari
       option :tags, default: proc { [] }
 
       def artifacts
-        results = search
-        return [] unless results
+        responses = search
+        return [] unless responses
 
-        flat_results = results.map do |result|
-          result["results"]
-        end.flatten.compact
-
-        flat_results.filter_map { |result| result["ip"] }.uniq
+        results = responses.map(&:results).flatten
+        results.map do |result|
+          build_artifact result
+        end
       end
 
       private
@@ -34,7 +34,8 @@ module Mihari
       end
 
       def search_with_page(query, page: 1)
-        api.simple.datascan(query, page: page)
+        res = api.simple.datascan(query, page: page)
+        Structs::Onyphe::Response.from_dynamic!(res)
       end
 
       def search
@@ -42,11 +43,35 @@ module Mihari
         (1..Float::INFINITY).each do |page|
           res = search_with_page(query, page: page)
           responses << res
-          total = res["total"].to_i
+
+          total = res.total
           break if total <= page * PAGE_SIZE
         end
         responses
       end
+
+      #
+      # Build an artifact from an Onyphe search API result
+      #
+      # @param [Structs::Onyphe::Result] result
+      #
+      # @return [Artifact]
+      #
+      def build_artifact(result)
+        as = AutonomousSystem.new(asn: normalize_asn(result.asn))
+
+        geolocation = Geolocation.new(
+          country: NormalizeCountry(result.country_code, to: :short),
+          country_code: result.country_code
+        )
+
+        Artifact.new(
+          data: result.ip,
+          source: source,
+          autonomous_system: as,
+          geolocation: geolocation
+        )
+      end
     end
   end
 end

data/lib/mihari/analyzers/shodan.rb

@@ -14,12 +14,11 @@ module Mihari
         results = search
         return [] unless results || results.empty?
 
+        results = results.map { |result| Structs::Shodan::Result.from_dynamic!(result) }
         results.map do |result|
-          matches = result["matches"] || []
-          matches.filter_map do |match|
-            match["ip_str"]
-          end
-        end.flatten.compact.uniq
+          matches = result.matches || []
+          matches.map { |match| build_artifact match }
+        end.flatten.compact.uniq(&:data)
       end
 
       private
@@ -57,6 +56,28 @@ module Mihari
         end
         responses
       end
+
+      #
+      # Build an artifact from a Shodan search API response
+      #
+      # @param [Structs::Shodan::Match] match
+      #
+      # @return [Artifact]
+      #
+      def build_artifact(match)
+        as = AutonomousSystem.new(asn: normalize_asn(match.asn))
+        geolocation = Geolocation.new(
+          country: match.location.country_name,
+          country_code: match.location.country_code
+        )
+
+        Artifact.new(
+          data: match.ip_str,
+          source: source,
+          autonomous_system: as,
+          geolocation: geolocation
+        )
+      end
     end
   end
 end

data/lib/mihari/database.rb

@@ -32,12 +32,48 @@ class InitialSchema < ActiveRecord::Migration[6.1]
   end
 end
 
-class V3Schema < ActiveRecord::Migration[6.1]
+class AddeSourceToArtifactSchema < ActiveRecord::Migration[6.1]
   def change
     add_column :artifacts, :source, :string, if_not_exists: true
   end
 end
 
+class EnrichmentsSchema < ActiveRecord::Migration[6.1]
+  def change
+    create_table :autonomous_systems, if_not_exists: true do |t|
+      t.integer :asn, null: false
+      t.belongs_to :artifact, foreign_key: true
+    end
+
+    create_table :geolocations, if_not_exists: true do |t|
+      t.string :country, null: false
+      t.string :country_code, null: false
+      t.belongs_to :artifact, foreign_key: true
+    end
+
+    create_table :whois_records, if_not_exists: true do |t|
+      t.string :domain, null: false
+      t.date :created_on
+      t.date :updated_on
+      t.date :expires_on
+      t.json :registrar
+      t.json :contacts
+      t.belongs_to :artifact, foreign_key: true
+    end
+
+    create_table :dns_records, if_not_exists: true do |t|
+      t.string :resource, null: false
+      t.string :value, null: false
+      t.belongs_to :artifact, foreign_key: true
+    end
+
+    create_table :reverse_dns_names, if_not_exists: true do |t|
+      t.string :name, null: false
+      t.belongs_to :artifact, foreign_key: true
+    end
+  end
+end
+
 def adapter
   return "postgresql" if Mihari.config.database.start_with?("postgresql://", "postgres://")
   return "mysql2" if Mihari.config.database.start_with?("mysql2://")
@@ -59,10 +95,12 @@ module Mihari
           )
         end
 
+        # ActiveRecord::Base.logger = Logger.new STDOUT
         ActiveRecord::Migration.verbose = false
 
         InitialSchema.migrate(:up)
-        V3Schema.migrate(:up)
+        AddeSourceToArtifactSchema.migrate(:up)
+        EnrichmentsSchema.migrate(:up)
       rescue StandardError
         # Do nothing
       end
@@ -76,7 +114,8 @@ module Mihari
         return unless ActiveRecord::Base.connected?
 
         InitialSchema.migrate(:down)
-        V3Schema.migrate(:down)
+        AddeSourceToArtifactSchema.migrate(:down)
+        EnrichmentsSchema.migrate(:down)
       end
     end
   end

data/lib/mihari/models/alert.rb

@@ -44,10 +44,12 @@ module Mihari
           to_at: to_at
         )
 
-        alerts = relation.limit(limit).offset(offset).order(id: :desc)
+        # TODO: improve queires
+        alert_ids = relation.limit(limit).offset(offset).order(id: :desc).pluck(:id).uniq
+        alerts = includes(:artifacts, :tags).where(id: [alert_ids]).order(id: :desc)
 
         alerts.map do |alert|
-          json = AlertSerializer.new(alert).as_json
+          json = Serializers::AlertSerializer.new(alert).as_json
           json[:artifacts] = json[:artifacts] || []
           json[:tags] = json[:tags] || []
           json
@@ -85,8 +87,10 @@ module Mihari
       def build_relation(artifact_data: nil, title: nil, description: nil, source: nil, tag_name: nil, from_at: nil, to_at: nil)
         relation = self
 
-        relation = relation.joins(:artifacts).where(artifacts: { data: artifact_data }) if artifact_data
-        relation = relation.joins(:tags).where(tags: { name: tag_name }) if tag_name
+        relation = relation.includes(:artifacts, :tags)
+
+        relation = relation.where(artifacts: { data: artifact_data }) if artifact_data
+        relation = relation.where(tags: { name: tag_name }) if tag_name
 
         relation = relation.where(source: source) if source
         relation = relation.where(title: title) if title

data/lib/mihari/models/artifact.rb

@@ -4,6 +4,7 @@ require "active_record"
 require "active_record/filter"
 require "active_support/core_ext/integer/time"
 require "active_support/core_ext/numeric/time"
+require "uri"
 
 class ArtifactValidator < ActiveModel::Validator
   def validate(record)
@@ -15,6 +16,13 @@ end
 
 module Mihari
   class Artifact < ActiveRecord::Base
+    has_one :autonomous_system, dependent: :destroy
+    has_one :geolocation, dependent: :destroy
+    has_one :whois_record, dependent: :destroy
+
+    has_many :dns_records, dependent: :destroy
+    has_many :reverse_dns_names, dependent: :destroy
+
     include ActiveModel::Validations
 
     validates_with ArtifactValidator
@@ -44,5 +52,52 @@ module Mihari
       #                           within {ignore_threshold} days, do not ignore it
       artifact.created_at < days_before
     end
+
+    #
+    # Enrich(add) whois record
+    #
+    def enrich_whois
+      return unless can_enrich_whois?
+
+      self.whois_record = WhoisRecord.build_by_domain(normalize_as_domain(data))
+    end
+
+    #
+    # Enrich(add) DNS records
+    #
+    def enrich_dns
+      return unless can_enrich_dns?
+
+      self.dns_records = DnsRecord.build_by_domain(normalize_as_domain(data))
+    end
+
+    #
+    # Enrich(add) reverse DNS names
+    #
+    def enrich_reverse_dns
+      return unless can_enrich_revese_dns?
+
+      self.reverse_dns_names = ReverseDnsName.build_by_ip(data)
+    end
+
+    private
+
+    def normalize_as_domain(url_or_domain)
+      return url_or_domain if data_type == "domain"
+
+      URI.parse(url_or_domain).host
+    end
+
+    def can_enrich_whois?
+      %w[domain url].include? data_type
+    end
+
+    def can_enrich_dns?
+      %w[domain url].include? data_type
+    end
+
+    def can_enrich_revese_dns?
+      data_type == "ip"
+    end
   end
 end