mihari 3.4.0 → 3.6.1

data/lib/mihari/cli/analyzer.rb

@@ -24,6 +24,7 @@ module Mihari
     class Analyzer < Base
       class_option :ignore_old_artifacts, type: :boolean, default: false, desc: "Whether to ignore old artifacts from checking or not."
       class_option :ignore_threshold, type: :numeric, default: 0, desc: "Number of days to define whether an artifact is old or not."
+      class_option :config, type: :string, desc: "Path to the config file"
 
       include Mihari::Commands::BinaryEdge
       include Mihari::Commands::Censys

data/lib/mihari/cli/base.rb

@@ -12,8 +12,6 @@ module Mihari
       include Mihari::Mixins::Hash
       include Mixins::Utils
 
-      class_option :config, type: :string, desc: "Path to the config file"
-
       class << self
         def exit_on_failure?
           true

data/lib/mihari/commands/init.rb

@@ -5,10 +5,10 @@ require "colorize"
 module Mihari
   module Commands
     module Initialization
-      def self.included(thor)
-        include Mixins::Configuration
-        include Mixins::Rule
+      include Mixins::Configuration
+      include Mixins::Rule
 
+      def self.included(thor)
         thor.class_eval do
           desc "config", "Create a config file"
           method_option :filename, type: :string, default: "mihari.yml"
@@ -37,7 +37,7 @@ module Mihari
 
             initialize_rule_yaml filename
 
-            puts "The rule file is created as #{filename}.".colorize(:blue)
+            puts "The rule file is initialized as #{filename}.".colorize(:blue)
           end
         end
       end

data/lib/mihari/commands/search.rb

@@ -8,6 +8,7 @@ module Mihari
       def self.included(thor)
         thor.class_eval do
           desc "search [RULE]", "Search by a rule"
+          method_option :config, type: :string, desc: "Path to the config file"
           def search_by_rule(rule)
             # convert str(YAML) to hash or str(path/YAML file) to hash
             rule = load_rule(rule)

data/lib/mihari/commands/web.rb

@@ -8,6 +8,7 @@ module Mihari
           desc "web", "Launch the web app"
           method_option :port, type: :numeric, default: 9292
           method_option :host, type: :string, default: "localhost"
+          method_option :config, type: :string, desc: "Path to the config file"
           def web
             port = options["port"].to_i || 9292
             host = options["host"] || "localhost"

data/lib/mihari/database.rb

@@ -32,12 +32,48 @@ class InitialSchema < ActiveRecord::Migration[6.1]
   end
 end
 
-class V3Schema < ActiveRecord::Migration[6.1]
+class AddeSourceToArtifactSchema < ActiveRecord::Migration[6.1]
   def change
     add_column :artifacts, :source, :string, if_not_exists: true
   end
 end
 
+class EnrichmentsSchema < ActiveRecord::Migration[6.1]
+  def change
+    create_table :autonomous_systems, if_not_exists: true do |t|
+      t.integer :asn, null: false
+      t.belongs_to :artifact, foreign_key: true
+    end
+
+    create_table :geolocations, if_not_exists: true do |t|
+      t.string :country, null: false
+      t.string :country_code, null: false
+      t.belongs_to :artifact, foreign_key: true
+    end
+
+    create_table :whois_records, if_not_exists: true do |t|
+      t.string :domain, null: false
+      t.date :created_on
+      t.date :updated_on
+      t.date :expires_on
+      t.json :registrar
+      t.json :contacts
+      t.belongs_to :artifact, foreign_key: true
+    end
+
+    create_table :dns_records, if_not_exists: true do |t|
+      t.string :resource, null: false
+      t.string :value, null: false
+      t.belongs_to :artifact, foreign_key: true
+    end
+
+    create_table :reverse_dns_names, if_not_exists: true do |t|
+      t.string :name, null: false
+      t.belongs_to :artifact, foreign_key: true
+    end
+  end
+end
+
 def adapter
   return "postgresql" if Mihari.config.database.start_with?("postgresql://", "postgres://")
   return "mysql2" if Mihari.config.database.start_with?("mysql2://")
@@ -59,10 +95,12 @@ module Mihari
           )
         end
 
+        # ActiveRecord::Base.logger = Logger.new STDOUT
         ActiveRecord::Migration.verbose = false
 
         InitialSchema.migrate(:up)
-        V3Schema.migrate(:up)
+        AddeSourceToArtifactSchema.migrate(:up)
+        EnrichmentsSchema.migrate(:up)
       rescue StandardError
         # Do nothing
       end
@@ -76,7 +114,8 @@ module Mihari
         return unless ActiveRecord::Base.connected?
 
         InitialSchema.migrate(:down)
-        V3Schema.migrate(:down)
+        AddeSourceToArtifactSchema.migrate(:down)
+        EnrichmentsSchema.migrate(:down)
       end
     end
   end

data/lib/mihari/emitters/base.rb

@@ -10,7 +10,7 @@ module Mihari
         Mihari.emitters << child
       end
 
-      # @return [true, false]
+      # @return [Boolean]
       def valid?
         raise NotImplementedError, "You must implement #{self.class}##{__method__}"
       end

data/lib/mihari/emitters/misp.rb

@@ -15,7 +15,7 @@ module Mihari
         end
       end
 
-      # @return [true, false]
+      # @return [Boolean]
       def valid?
         api_endpoint? && api_key? && ping?
       end
@@ -42,10 +42,24 @@ module Mihari
         %w[misp_api_endpoint misp_api_key]
       end
 
+      #
+      # Build a MISP attribute
+      #
+      # @param [Mihari::Artifact] artifact
+      #
+      # @return [::MISP::Attribute] <description>
+      #
       def build_attribute(artifact)
         ::MISP::Attribute.new(value: artifact.data, type: to_misp_type(type: artifact.data_type, value: artifact.data))
       end
 
+      #
+      # Get a type of a hash
+      #
+      # @param [String] value
+      #
+      # @return [String]
+      #
       def hash_type(value)
         case value.length
         when 32
@@ -59,6 +73,14 @@ module Mihari
         end
       end
 
+      #
+      # Convert a type to a MISP type
+      #
+      # @param [String] type
+      # @param [String] value
+      #
+      # @return [String]
+      #
       def to_misp_type(type:, value:)
         type = type.to_sym
         table = {
@@ -72,20 +94,31 @@ module Mihari
         hash_type value
       end
 
-      def create_attribute(artifact)
-        artifact.data_type
-      end
-
+      #
+      # Check whether an API endpoint is set or not
+      #
+      # @return [Boolean]
+      #
       def api_endpoint?
         api_endpoint = ::MISP.configuration.api_endpoint
         !api_endpoint.nil? && !api_endpoint.empty?
       end
 
+      #
+      # Check whether an API key is set or not
+      #
+      # @return [Boolean]
+      #
       def api_key?
         api_key = ::MISP.configuration.api_key
         !api_key.nil? && !api_key.empty?
       end
 
+      #
+      # Check whether an API endpoint is reachable or not
+      #
+      # @return [Boolean]
+      #
       def ping?
         base_url = ::MISP.configuration.api_endpoint
         base_url = base_url.end_with?("/") ? base_url[0..-2] : base_url

data/lib/mihari/emitters/slack.rb

@@ -55,7 +55,7 @@ module Mihari
 
       private
 
-      # @return [String]
+      # @return [String, nil]
       def _urlscan_link
         case data_type
         when "ip"
@@ -69,7 +69,7 @@ module Mihari
       end
       memoize :_urlscan_link
 
-      # @return [String]
+      # @return [String, nil]
       def _vt_link
         case data_type
         when "hash"
@@ -86,11 +86,13 @@ module Mihari
       end
       memoize :_vt_link
 
+      # @return [String, nil]
       def _censys_link
         data_type == "ip" ? "https://search.censys.io/hosts/#{data}" : nil
       end
       memoize :_censys_link
 
+      # @return [String, nil]
       def _shodan_link
         data_type == "ip" ? "https://www.shodan.io/host/#{data}" : nil
       end
@@ -116,12 +118,28 @@ module Mihari
         notifier.valid?
       end
 
+      #
+      # Build attachements
+      #
+      # @param [Array<Mihari::Artifact>] artifacts
+      #
+      # @return [Array<Mihari::Emitters::Attachment>]
+      #
       def to_attachments(artifacts)
         artifacts.map do |artifact|
           Attachment.new(data: artifact.data, data_type: artifact.data_type).to_a
         end.flatten
       end
 
+      #
+      # Build a text
+      #
+      # @param [String] title
+      # @param [String] description
+      # @param [Array<String>] tags
+      #
+      # @return [String]
+      #
       def to_text(title:, description:, tags: [])
         tags = ["N/A"] if tags.empty?
 

data/lib/mihari/emitters/the_hive.rb

@@ -6,7 +6,7 @@ require "net/ping"
 module Mihari
   module Emitters
     class TheHive < Base
-      # @return [true, false]
+      # @return [Boolean]
       def valid?
         api_endpont? && api_key? && ping?
       end
@@ -34,16 +34,29 @@ module Mihari
         @api ||= Hachi::API.new(api_endpoint: Mihari.config.thehive_api_endpoint, api_key: Mihari.config.thehive_api_key)
       end
 
-      # @return [true, false]
+      #
+      # Check whether an API endpoint is set or not
+      #
+      # @return [Boolean]
+      #
       def api_endpont?
         !Mihari.config.thehive_api_endpoint.nil?
       end
 
-      # @return [true, false]
+      #
+      # Check whether an API key is set or not
+      #
+      # @return [Boolean]
+      # ]
       def api_key?
         !Mihari.config.thehive_api_key.nil?
       end
 
+      #
+      # Check whether an API endpoint is reachable or not
+      #
+      # @return [Boolean]
+      #
       def ping?
         base_url = Mihari.config.thehive_api_endpoint
         base_url = base_url.end_with?("/") ? base_url[0..-2] : base_url

data/lib/mihari/emitters/webhook.rb

@@ -7,7 +7,7 @@ require "uri"
 module Mihari
   module Emitters
     class Webhook < Base
-      # @return [true, false]
+      # @return [Boolean]
       def valid?
         webhook_url?
       end
@@ -24,7 +24,7 @@ module Mihari
           tags: tags
         }
 
-        if use_json_body
+        if use_json_body?
           Net::HTTP.post(uri, data.to_json, "Content-Type" => "application/json")
         else
           Net::HTTP.post_form(uri, data)
@@ -37,15 +37,30 @@ module Mihari
         %w[webhook_url]
       end
 
+      #
+      # Webhook URL
+      #
+      # @return [String, nil]
+      #
       def webhook_url
         @webhook_url ||= Mihari.config.webhook_url
       end
 
+      #
+      # Check whether a webhook URL is set or not
+      #
+      # @return [<Type>] <description>
+      #
       def webhook_url?
         !webhook_url.nil?
       end
 
-      def use_json_body
+      #
+      # Check whether to use JSON body or NOT
+      #
+      # @return [<Type>] <description>
+      #
+      def use_json_body?
         @use_json_body ||= Mihari.config.webhook_use_json_body
       end
     end

data/lib/mihari/mixins/disallowed_data_value.rb

@@ -17,7 +17,7 @@ module Mihari
 
         # if a value is surrounded by slashes, take it as a regexp
         value_without_slashes = value[1..-2]
-        Regexp.compile value_without_slashes
+        Regexp.compile value_without_slashes.to_s
       end
 
       memoize :normalize_disallowed_data_value

data/lib/mihari/models/alert.rb

@@ -18,8 +18,8 @@ module Mihari
       # @param [String, nil] source
       # @param [String, nil] tag_name
       # @param [String, nil] title
-      # @param [String, nil] from_at
-      # @param [String, nil] to_at
+      # @param [DateTime, nil] from_at
+      # @param [DateTime, nil] to_at
       # @param [Integer, nil] limit
       # @param [Integer, nil] page
       #
@@ -34,12 +34,22 @@ module Mihari
 
         offset = (page - 1) * limit
 
-        relation = build_relation(artifact_data: artifact_data, title: title, description: description, source: source, tag_name: tag_name, from_at: from_at, to_at: to_at)
+        relation = build_relation(
+          artifact_data: artifact_data,
+          title: title,
+          description: description,
+          source: source,
+          tag_name: tag_name,
+          from_at: from_at,
+          to_at: to_at
+        )
 
-        alerts = relation.limit(limit).offset(offset).order(id: :desc)
+        # TODO: improve queires
+        alert_ids = relation.limit(limit).offset(offset).order(id: :desc).pluck(:id).uniq
+        alerts = includes(:artifacts, :tags).where(id: [alert_ids]).order(id: :desc)
 
         alerts.map do |alert|
-          json = AlertSerializer.new(alert).as_json
+          json = Serializers::AlertSerializer.new(alert).as_json
           json[:artifacts] = json[:artifacts] || []
           json[:tags] = json[:tags] || []
           json
@@ -54,13 +64,21 @@ module Mihari
       # @param [String, nil] source
       # @param [String, nil] tag_name
       # @param [String, nil] title
-      # @param [String, nil] from_at
-      # @param [String, nil] to_at
+      # @param [DateTime, nil] from_at
+      # @param [DateTime, nil] to_at
       #
       # @return [Integer]
       #
       def count(artifact_data: nil, description: nil, source: nil, tag_name: nil, title: nil, from_at: nil, to_at: nil)
-        relation = build_relation(artifact_data: artifact_data, title: title, description: description, source: source, tag_name: tag_name, from_at: from_at, to_at: to_at)
+        relation = build_relation(
+          artifact_data: artifact_data,
+          title: title,
+          description: description,
+          source: source,
+          tag_name: tag_name,
+          from_at: from_at,
+          to_at: to_at
+        )
         relation.distinct("alerts.id").count
       end
 
@@ -68,8 +86,8 @@ module Mihari
 
       def build_relation(artifact_data: nil, title: nil, description: nil, source: nil, tag_name: nil, from_at: nil, to_at: nil)
         relation = self
-        relation = joins(:tags) if tag_name
-        relation = joins(:artifacts) if artifact_data
+
+        relation = relation.includes(:artifacts, :tags)
 
         relation = relation.where(artifacts: { data: artifact_data }) if artifact_data
         relation = relation.where(tags: { name: tag_name }) if tag_name