mihari 3.3.0 → 3.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 56538b2cc9269fa7e92a2c8cd98746caae7a480b7961fb83482a2189f5318992
-  data.tar.gz: 9c4561c9e820a42b9b4d65b1e0ad02901034cc7eb124d7c5409418cab43bc5bd
+  metadata.gz: c9c1cbdf0570c25e2d89d7f6fd402b64991dfaebc75cf3cf5422a56504287ae9
+  data.tar.gz: d5b7a0db7b49f245e3c949135011fb674e28a9d3c251e165f827e8f3d90673b1
 SHA512:
-  metadata.gz: 0b43cf2ee5e73607593e2c140dbc4ec70dcc8ad1eb436914a016f852f8dabed7f1aafd2f048565356f3fbed865540488e0766bc7f3a86bd6845c722419472abe
-  data.tar.gz: 485bbcd01bcd8016b3a50b5e61fcac4cef8f527812570c734eff8a0fe6a75fc3b8e79e50d2a147c4edc20eee0370609ec5c2021bd28172bedff484bd04707379
+  metadata.gz: e76a216dedbc1aec17748c37a1b874c2c825fed6f7716ef356a48ddf2861584da299c384737e588a48b67165874f495192bb42a4c20c2f29f4620f8b559d1a83
+  data.tar.gz: 2f3e380b252ba238594ccacd2df8e362a62b9685aafeff34d6506e7301184cf5b38c4244db9498842f4aee9df109d7c43a9ad99cecc3b63616d333a7f5093333

data/README.md

@@ -53,6 +53,10 @@ Mihari supports the following services by default.
 
 - [Mihari Knowledge Base](https://www.notion.so/Mihari-Knowledge-Base-266994ff61204428ba6cfcebe40b0bd1)
 
+## Presentations
+
+- [Adversary Infrastructure Tracking with Mihari](https://ninoseki.github.io/presentations/Adversary%20Infrastructure%20Tracking%20with%20Mihari.pdf)
+
 ## License
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/config.ru

@@ -1,3 +1,4 @@
+# bundle exec rerun -- rackup config.ru
 require "./lib/mihari"
 
 # set rack env as development

data/lib/mihari/analyzers/base.rb

@@ -51,7 +51,7 @@ module Mihari
       # @return [nil]
       #
       def run
-        set_unique_artifacts
+        set_enriched_artifacts
 
         Parallel.each(valid_emitters) do |emitter|
           run_emitter emitter
@@ -66,7 +66,7 @@ module Mihari
       # @return [nil]
       #
       def run_emitter(emitter)
-        emitter.run(title: title, description: description, artifacts: unique_artifacts, source: source, tags: tags)
+        emitter.run(title: title, description: description, artifacts: enriched_artifacts, source: source, tags: tags)
       rescue StandardError => e
         puts "Emission by #{emitter.class} is failed: #{e}"
       end
@@ -88,7 +88,7 @@ module Mihari
           # No need to set data_type manually
           # It is set automatically in #initialize
           artifact.is_a?(Artifact) ? artifact : Artifact.new(data: artifact, source: source)
-        end.select(&:valid?)
+        end.select(&:valid?).uniq(&:data)
       end
 
       private
@@ -105,12 +105,26 @@ module Mihari
       end
 
       #
-      # Set unique artifacts
+      # Enriched artifacts
+      #
+      # @return [Array<Mihari::Artifact>]
+      #
+      def enriched_artifacts
+        @enriched_artifacts ||= unique_artifacts.map do |artifact|
+          artifact.enrich_whois
+          artifact.enrich_dns
+          artifact.enrich_reverse_dns
+          artifact
+        end
+      end
+
+      #
+      # Set enriched artifacts
       #
       # @return [nil]
       #
-      def set_unique_artifacts
-        retry_on_error { unique_artifacts }
+      def set_enriched_artifacts
+        retry_on_error { enriched_artifacts }
       rescue ArgumentError => _e
         klass = self.class.to_s.split("::").last.to_s
         raise Error, "Please configure #{klass} API settings properly"
@@ -127,6 +141,20 @@ module Mihari
           emitter.valid? ? emitter : nil
         end
       end
+
+      #
+      # Normalize ASN value
+      #
+      # @param [String, Integer] asn
+      #
+      # @return [Integer]
+      #
+      def normalize_asn(asn)
+        return asn if asn.is_a?(Integer)
+        return asn.to_i unless asn.start_with?("AS")
+
+        asn.delete_prefix("AS").to_i
+      end
     end
   end
 end

data/lib/mihari/analyzers/censys.rb

@@ -17,31 +17,59 @@ module Mihari
       private
 
       def search
-        ipv4s = []
+        artifacts = []
 
         cursor = nil
         loop do
           response = api.search(query, cursor: cursor)
-          ipv4s << response_to_ipv4s(response)
+          response = Structs::Censys::Response.from_dynamic!(response)
 
-          links = response.dig("result", "links")
-          cursor = links["next"]
+          artifacts << response_to_artifacts(response)
+
+          cursor = response.result.links.next
           break if cursor == ""
         end
 
-        ipv4s.flatten
+        artifacts.flatten.uniq(&:data)
       end
 
       #
       # Extract IPv4s from Censys search API response
       #
-      # @param [Hash] response
+      # @param [Structs::Censys::Response] response
       #
       # @return [Array<String>]
       #
-      def response_to_ipv4s(response)
-        hits = response.dig("result", "hits") || []
-        hits.map { |hit| hit["ip"] }
+      def response_to_artifacts(response)
+        response.result.hits.map { |hit| build_artifact(hit) }
+      end
+
+      #
+      # Build an artifact from a Shodan search API response
+      #
+      # @param [Structs::Censys::Hit] hit
+      #
+      # @return [Artifact]
+      #
+      def build_artifact(hit)
+        as = AutonomousSystem.new(asn: normalize_asn(hit.autonomous_system.asn))
+
+        # sometimes Censys overlooks country
+        # then set geolocation as nil
+        geolocation = nil
+        unless hit.location.country.nil?
+          geolocation = Geolocation.new(
+            country: hit.location.country,
+            country_code: hit.location.country_code
+          )
+        end
+
+        Artifact.new(
+          data: hit.ip,
+          source: source,
+          autonomous_system: as,
+          geolocation: geolocation
+        )
       end
 
       def configuration_keys

data/lib/mihari/analyzers/onyphe.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "onyphe"
+require "normalize_country"
 
 module Mihari
   module Analyzers
@@ -11,14 +12,13 @@ module Mihari
       option :tags, default: proc { [] }
 
       def artifacts
-        results = search
-        return [] unless results
+        responses = search
+        return [] unless responses
 
-        flat_results = results.map do |result|
-          result["results"]
-        end.flatten.compact
-
-        flat_results.filter_map { |result| result["ip"] }.uniq
+        results = responses.map(&:results).flatten
+        results.map do |result|
+          build_artifact result
+        end
       end
 
       private
@@ -34,7 +34,8 @@ module Mihari
       end
 
       def search_with_page(query, page: 1)
-        api.simple.datascan(query, page: page)
+        res = api.simple.datascan(query, page: page)
+        Structs::Onyphe::Response.from_dynamic!(res)
       end
 
       def search
@@ -42,11 +43,35 @@ module Mihari
         (1..Float::INFINITY).each do |page|
           res = search_with_page(query, page: page)
           responses << res
-          total = res["total"].to_i
+
+          total = res.total
           break if total <= page * PAGE_SIZE
         end
         responses
       end
+
+      #
+      # Build an artifact from an Onyphe search API result
+      #
+      # @param [Structs::Onyphe::Result] result
+      #
+      # @return [Artifact]
+      #
+      def build_artifact(result)
+        as = AutonomousSystem.new(asn: normalize_asn(result.asn))
+
+        geolocation = Geolocation.new(
+          country: NormalizeCountry(result.country_code, to: :short),
+          country_code: result.country_code
+        )
+
+        Artifact.new(
+          data: result.ip,
+          source: source,
+          autonomous_system: as,
+          geolocation: geolocation
+        )
+      end
     end
   end
 end

data/lib/mihari/analyzers/shodan.rb

@@ -14,12 +14,11 @@ module Mihari
         results = search
         return [] unless results || results.empty?
 
+        results = results.map { |result| Structs::Shodan::Result.from_dynamic!(result) }
         results.map do |result|
-          matches = result["matches"] || []
-          matches.filter_map do |match|
-            match["ip_str"]
-          end
-        end.flatten.compact.uniq
+          matches = result.matches || []
+          matches.map { |match| build_artifact match }
+        end.flatten.compact.uniq(&:data)
       end
 
       private
@@ -57,6 +56,28 @@ module Mihari
         end
         responses
       end
+
+      #
+      # Build an artifact from a Shodan search API response
+      #
+      # @param [Structs::Shodan::Match] match
+      #
+      # @return [Artifact]
+      #
+      def build_artifact(match)
+        as = AutonomousSystem.new(asn: normalize_asn(match.asn))
+        geolocation = Geolocation.new(
+          country: match.location.country_name,
+          country_code: match.location.country_code
+        )
+
+        Artifact.new(
+          data: match.ip_str,
+          source: source,
+          autonomous_system: as,
+          geolocation: geolocation
+        )
+      end
     end
   end
 end

data/lib/mihari/cli/analyzer.rb

@@ -22,6 +22,10 @@ require "mihari/commands/json"
 module Mihari
   module CLI
     class Analyzer < Base
+      class_option :ignore_old_artifacts, type: :boolean, default: false, desc: "Whether to ignore old artifacts from checking or not."
+      class_option :ignore_threshold, type: :numeric, default: 0, desc: "Number of days to define whether an artifact is old or not."
+      class_option :config, type: :string, desc: "Path to the config file"
+
       include Mihari::Commands::BinaryEdge
       include Mihari::Commands::Censys
       include Mihari::Commands::CIRCL

data/lib/mihari/cli/base.rb

@@ -12,11 +12,6 @@ module Mihari
       include Mihari::Mixins::Hash
       include Mixins::Utils
 
-      class_option :config, type: :string, desc: "Path to the config file"
-
-      class_option :ignore_old_artifacts, type: :boolean, default: false, desc: "Whether to ignore old artifacts from checking or not. Only affects with analyze commands."
-      class_option :ignore_threshold, type: :numeric, default: 0, desc: "Number of days to define whether an artifact is old or not. Only affects with analyze commands."
-
       class << self
         def exit_on_failure?
           true

data/lib/mihari/commands/init.rb

@@ -5,10 +5,10 @@ require "colorize"
 module Mihari
   module Commands
     module Initialization
-      def self.included(thor)
-        include Mixins::Configuration
-        include Mixins::Rule
+      include Mixins::Configuration
+      include Mixins::Rule
 
+      def self.included(thor)
         thor.class_eval do
           desc "config", "Create a config file"
           method_option :filename, type: :string, default: "mihari.yml"
@@ -37,7 +37,7 @@ module Mihari
 
             initialize_rule_yaml filename
 
-            puts "The rule file is created as #{filename}.".colorize(:blue)
+            puts "The rule file is initialized as #{filename}.".colorize(:blue)
           end
         end
       end

data/lib/mihari/commands/search.rb

@@ -8,17 +8,27 @@ module Mihari
       def self.included(thor)
         thor.class_eval do
           desc "search [RULE]", "Search by a rule"
+          method_option :config, type: :string, desc: "Path to the config file"
           def search_by_rule(rule)
             # convert str(YAML) to hash or str(path/YAML file) to hash
             rule = load_rule(rule)
 
             # validate rule schema
-            validate_rule rule
+            rule = validate_rule(rule)
 
-            analyzer = build_rule_analyzer(**rule)
+            analyzer = build_rule_analyzer(
+              title: rule[:title],
+              description: rule[:description],
+              queries: rule[:queries],
+              tags: rule[:tags],
+              allowed_data_types: rule[:allowed_data_types],
+              disallowed_data_values: rule[:disallowed_data_values],
+              source: rule[:source],
+              id: rule[:id]
+            )
 
-            ignore_old_artifacts = options["ignore_old_artifacts"] || false
-            ignore_threshold = options["ignore_threshold"] || 0
+            ignore_old_artifacts = rule[:ignore_old_artifacts]
+            ignore_threshold = rule[:ignore_threshold]
 
             with_error_handling do
               run_rule_analyzer analyzer, ignore_old_artifacts: ignore_old_artifacts, ignore_threshold: ignore_threshold
@@ -42,7 +52,7 @@ module Mihari
       #
       # @return [Mihari::Analyzers::Rule]
       #
-      def build_rule_analyzer(title:, description:, queries:, tags: nil, allowed_data_types: nil, disallowed_data_values: nil, source: nil)
+      def build_rule_analyzer(title:, description:, queries:, tags: nil, allowed_data_types: nil, disallowed_data_values: nil, source: nil, id: nil)
         tags = [] if tags.nil?
         allowed_data_types = ALLOWED_DATA_TYPES if allowed_data_types.nil?
         disallowed_data_values = [] if disallowed_data_values.nil?
@@ -54,7 +64,8 @@ module Mihari
           queries: queries,
           allowed_data_types: allowed_data_types,
           disallowed_data_values: disallowed_data_values,
-          source: source
+          source: source,
+          id: id
         )
       end
 
@@ -62,8 +73,6 @@ module Mihari
       # Run rule analyzer
       #
       # @param [Mihari::Analyzer::Rule] analyzer
-      # @param [Boolean] ignore_old_artifacts
-      # @param [Integer] ignore_threshold
       #
       # @return [nil]
       #

data/lib/mihari/commands/web.rb

@@ -8,6 +8,7 @@ module Mihari
           desc "web", "Launch the web app"
           method_option :port, type: :numeric, default: 9292
           method_option :host, type: :string, default: "localhost"
+          method_option :config, type: :string, desc: "Path to the config file"
           def web
             port = options["port"].to_i || 9292
             host = options["host"] || "localhost"

data/lib/mihari/database.rb

@@ -32,12 +32,48 @@ class InitialSchema < ActiveRecord::Migration[6.1]
   end
 end
 
-class V3Schema < ActiveRecord::Migration[6.1]
+class AddeSourceToArtifactSchema < ActiveRecord::Migration[6.1]
   def change
     add_column :artifacts, :source, :string, if_not_exists: true
   end
 end
 
+class EnrichmentsSchema < ActiveRecord::Migration[6.1]
+  def change
+    create_table :autonomous_systems, if_not_exists: true do |t|
+      t.integer :asn, null: false
+      t.belongs_to :artifact, foreign_key: true
+    end
+
+    create_table :geolocations, if_not_exists: true do |t|
+      t.string :country, null: false
+      t.string :country_code, null: false
+      t.belongs_to :artifact, foreign_key: true
+    end
+
+    create_table :whois_records, if_not_exists: true do |t|
+      t.string :domain, null: false
+      t.date :created_on
+      t.date :updated_on
+      t.date :expires_on
+      t.json :registrar
+      t.json :contacts
+      t.belongs_to :artifact, foreign_key: true
+    end
+
+    create_table :dns_records, if_not_exists: true do |t|
+      t.string :resource, null: false
+      t.string :value, null: false
+      t.belongs_to :artifact, foreign_key: true
+    end
+
+    create_table :reverse_dns_names, if_not_exists: true do |t|
+      t.string :name, null: false
+      t.belongs_to :artifact, foreign_key: true
+    end
+  end
+end
+
 def adapter
   return "postgresql" if Mihari.config.database.start_with?("postgresql://", "postgres://")
   return "mysql2" if Mihari.config.database.start_with?("mysql2://")
@@ -59,10 +95,12 @@ module Mihari
           )
         end
 
+        # ActiveRecord::Base.logger = Logger.new STDOUT
         ActiveRecord::Migration.verbose = false
 
         InitialSchema.migrate(:up)
-        V3Schema.migrate(:up)
+        AddeSourceToArtifactSchema.migrate(:up)
+        EnrichmentsSchema.migrate(:up)
       rescue StandardError
         # Do nothing
       end
@@ -76,7 +114,8 @@ module Mihari
         return unless ActiveRecord::Base.connected?
 
         InitialSchema.migrate(:down)
-        V3Schema.migrate(:down)
+        AddeSourceToArtifactSchema.migrate(:down)
+        EnrichmentsSchema.migrate(:down)
       end
     end
   end