mihari 1.3.2 → 2.0.0

data/lib/mihari/cli.rb

@@ -1,7 +1,9 @@
 # frozen_string_literal: true
 
-require "thor"
 require "json"
+require "rack/builder"
+require "rack/handler/webrick"
+require "thor"
 
 module Mihari
   class CLI < Thor
@@ -46,7 +48,10 @@ module Mihari
     method_option :title, type: :string, desc: "title"
     method_option :description, type: :string, desc: "description"
     method_option :tags, type: :array, desc: "tags"
+    method_option :filter, type: :string, desc: "filter for urlscan pro search"
     method_option :target_type, type: :string, default: "url", desc: "target type to fetch from lookup results (target type should be 'url', 'domain' or 'ip')"
+    method_option :use_pro, type: :boolean, default: false, desc: "use pro search API or not"
+    method_option :use_similarity, type: :boolean, default: false, desc: "use similarity API or not"
     def urlscan(query)
       with_error_handling do
         run_analyzer Analyzers::Urlscan, query: query, options: options
@@ -256,44 +261,31 @@ module Mihari
     desc "import_from_json", "Give a JSON input via STDIN"
     def import_from_json(input = nil)
       with_error_handling do
-        json = input || STDIN.gets.chomp
+        json = input || $stdin.gets.chomp
         raise ArgumentError, "Input not found: please give an input in a JSON format" unless json
 
         json = parse_as_json(json)
         raise ArgumentError, "Invalid input format: an input JSON data should have title, description and artifacts key" unless valid_json?(json)
 
-        title = json.dig("title")
-        description = json.dig("description")
-        artifacts = json.dig("artifacts")
-        tags = json.dig("tags") || []
+        title = json["title"]
+        description = json["description"]
+        artifacts = json["artifacts"]
+        tags = json["tags"] || []
 
         basic = Analyzers::Basic.new(title: title, description: description, artifacts: artifacts, source: "json", tags: tags)
         basic.run
       end
     end
 
-    desc "alerts", "Show the alerts on TheHive"
-    method_option :limit, type: :string, default: "5", desc: "Number of alerts to show (or 'all' to show all the alerts)"
-    method_option :title, type: :string, desc: "Title to filter"
-    method_option :source, type: :string, desc: "Source to filter"
-    method_option :tag, type: :string, desc: "Tag to filter"
-    def alerts
-      with_error_handling do
-        load_configuration
+    desc "web", "Launch the web app"
+    method_option :port, type: :numeric, default: 9292
+    method_option :host, type: :string, default: "localhost"
+    def web
+      port = options["port"].to_i || 9292
+      host = options["host"] || "localhost"
 
-        viewer = AlertViewer.new
-        alerts = viewer.list(limit: options["limit"], title: options["title"], source: options["source"], tag: options[:tag])
-        puts JSON.pretty_generate(alerts)
-      end
-    end
-
-    desc "status", "Show the current configuration status"
-    def status
-      with_error_handling do
-        load_configuration
-
-        puts JSON.pretty_generate(Status.check)
-      end
+      load_configuration
+      Mihari::App.run!(port: port, host: host)
     end
 
     no_commands do
@@ -312,7 +304,7 @@ module Mihari
 
       # @return [true, false]
       def valid_json?(json)
-        %w(title description artifacts).all? { |key| json.key? key }
+        %w[title description artifacts].all? { |key| json.key? key }
       end
 
       def load_configuration

data/lib/mihari/config.rb

@@ -4,31 +4,7 @@ require "yaml"
 
 module Mihari
   class Config
-    attr_accessor :binaryedge_api_key
-    attr_accessor :censys_id
-    attr_accessor :censys_secret
-    attr_accessor :circl_passive_password
-    attr_accessor :circl_passive_username
-    attr_accessor :misp_api_endpoint
-    attr_accessor :misp_api_key
-    attr_accessor :onyphe_api_key
-    attr_accessor :otx_api_key
-    attr_accessor :passivetotal_api_key
-    attr_accessor :passivetotal_username
-    attr_accessor :pulsedive_api_key
-    attr_accessor :securitytrails_api_key
-    attr_accessor :shodan_api_key
-    attr_accessor :slack_channel
-    attr_accessor :slack_webhook_url
-    attr_accessor :spyse_api_key
-    attr_accessor :thehive_api_endpoint
-    attr_accessor :thehive_api_key
-    attr_accessor :urlscan_api_key
-    attr_accessor :virustotal_api_key
-    attr_accessor :zoomeye_password
-    attr_accessor :zoomeye_username
-
-    attr_accessor :database
+    attr_accessor :binaryedge_api_key, :censys_id, :censys_secret, :circl_passive_password, :circl_passive_username, :misp_api_endpoint, :misp_api_key, :onyphe_api_key, :otx_api_key, :passivetotal_api_key, :passivetotal_username, :pulsedive_api_key, :securitytrails_api_key, :shodan_api_key, :slack_channel, :slack_webhook_url, :spyse_api_key, :thehive_api_endpoint, :thehive_api_key, :urlscan_api_key, :virustotal_api_key, :zoomeye_password, :zoomeye_username, :database
 
     def initialize
       load_from_env

data/lib/mihari/configurable.rb

@@ -6,13 +6,12 @@ module Mihari
       config_keys.all? { |key| Mihari.config.send(key) }
     end
 
-    def configuration_status
+    def configuration_values
       return nil if config_keys.empty?
 
-      names = config_keys.join(" and ")
-      be_verb = config_keys.length == 1 ? "is" : "are"
-      status = configured? ? "found" : "missing"
-      "#{names} #{be_verb} #{status}"
+      config_keys.map do |key|
+        {key: key.upcase, value: Mihari.config.send(key)}
+      end
     end
 
     def config_keys

data/lib/mihari/database.rb

@@ -34,6 +34,7 @@ end
 
 def adapter
   return "postgresql" if Mihari.config.database.start_with?("postgresql://", "postgres://")
+  return "mysql2" if Mihari.config.database.start_with?("mysql2://")
 
   "sqlite3"
 end
@@ -43,7 +44,7 @@ module Mihari
     class << self
       def connect
         case adapter
-        when "postgresql"
+        when "postgresql", "mysql2"
           ActiveRecord::Base.establish_connection(Mihari.config.database)
         else
           ActiveRecord::Base.establish_connection(
@@ -58,6 +59,11 @@ module Mihari
         # Do nothing
       end
 
+      def close
+        ActiveRecord::Base.clear_active_connections!
+        ActiveRecord::Base.connection.close
+      end
+
       def destroy!
         InitialSchema.migrate(:down) if ActiveRecord::Base.connected?
       end

data/lib/mihari/emitters/misp.rb

@@ -7,6 +7,8 @@ module Mihari
   module Emitters
     class MISP < Base
       def initialize
+        super()
+
         ::MISP.configure do |config|
           config.api_endpoint = Mihari.config.misp_api_endpoint
           config.api_key = Mihari.config.misp_api_key
@@ -35,7 +37,7 @@ module Mihari
       private
 
       def config_keys
-        %w(misp_api_endpoint misp_api_key)
+        %w[misp_api_endpoint misp_api_key]
       end
 
       def build_attribute(artifact)
@@ -61,7 +63,7 @@ module Mihari
           ip: "ip-dst",
           mail: "email-dst",
           url: "url",
-          domain: "domain",
+          domain: "domain"
         }
         return table[type] if table.key?(type)
 

data/lib/mihari/emitters/slack.rb

@@ -19,25 +19,31 @@ module Mihari
       end
 
       def actions
-        [vt_link, urlscan_link, censys_link].compact
+        [vt_link, urlscan_link, censys_link, shodan_link].compact
       end
 
       def vt_link
         return nil unless _vt_link
 
-        { type: "button", text: "Lookup on VirusTotal", url: _vt_link, }
+        {type: "button", text: "VirusTotal", url: _vt_link}
       end
 
       def urlscan_link
         return nil unless _urlscan_link
 
-        { type: "button", text: "Lookup on urlscan.io", url: _urlscan_link, }
+        {type: "button", text: "urlscan.io", url: _urlscan_link}
       end
 
       def censys_link
         return nil unless _censys_link
 
-        { type: "button", text: "Lookup on Censys", url: _censys_link, }
+        {type: "button", text: "Censys", url: _censys_link}
+      end
+
+      def shodan_link
+        return nil unless _shodan_link
+
+        {type: "button", text: "Shodan", url: _shodan_link}
       end
 
       # @return [Array]
@@ -89,6 +95,11 @@ module Mihari
       end
       memoize :_censys_link
 
+      def _shodan_link
+        data_type == "ip" ? "https://www.shodan.io/host/#{data}" : nil
+      end
+      memoize :_shodan_link
+
       # @return [String]
       def sha256
         Digest::SHA256.hexdigest data
@@ -96,7 +107,7 @@ module Mihari
 
       # @return [String]
       def defanged_data
-        @defanged_data ||= data.to_s.gsub /\./, "[.]"
+        @defanged_data ||= data.to_s.gsub(/\./, "[.]")
       end
     end
 
@@ -121,7 +132,7 @@ module Mihari
         [
           "*#{title}*",
           "*Desc.*: #{description}",
-          "*Tags*: #{tags.join(', ')}",
+          "*Tags*: #{tags.join(", ")}"
         ].join("\n")
       end
 
@@ -137,7 +148,7 @@ module Mihari
       private
 
       def config_keys
-        %w(slack_webhook_url)
+        %w[slack_webhook_url]
       end
     end
   end

data/lib/mihari/emitters/the_hive.rb

@@ -17,7 +17,7 @@ module Mihari
         api.alert.create(
           title: title,
           description: description,
-          artifacts: artifacts.map { |artifact| { data: artifact.data, data_type: artifact.data_type, message: description } },
+          artifacts: artifacts.map { |artifact| {data: artifact.data, data_type: artifact.data_type, message: description} },
           tags: tags,
           type: "external",
           source: "mihari"
@@ -27,7 +27,7 @@ module Mihari
       private
 
       def config_keys
-        %w(thehive_api_endpoint thehive_api_key)
+        %w[thehive_api_endpoint thehive_api_key]
       end
 
       def api

data/lib/mihari/errors.rb

@@ -2,6 +2,8 @@
 
 module Mihari
   class Error < StandardError; end
+
   class InvalidInputError < Error; end
+
   class RetryableError < Error; end
 end

data/lib/mihari/models/alert.rb

@@ -1,11 +1,62 @@
 # frozen_string_literal: true
 
 require "active_record"
+require "active_record/filter"
 
 module Mihari
   class Alert < ActiveRecord::Base
     has_many :taggings, dependent: :destroy
     has_many :artifacts, dependent: :destroy
     has_many :tags, through: :taggings
+
+    class << self
+      def search(artifact_data: nil, description: nil, source: nil, tag_name: nil, title: nil, from_at: nil, to_at: nil, limit: 10, page: 1)
+        limit = limit.to_i
+        raise ArgumentError, "limit should be bigger than zero" unless limit.positive?
+
+        page = page.to_i
+        raise ArgumentError, "page should be bigger than zero" unless page.positive?
+
+        offset = (page - 1) * limit
+
+        relation = build_relation(artifact_data: artifact_data, title: title, description: description, source: source, tag_name: tag_name, from_at: from_at, to_at: to_at)
+        # relation = relation.group("alerts.id")
+
+        alerts = relation.limit(limit).offset(offset).order(id: :desc)
+
+        alerts.map do |alert|
+          json = AlertSerializer.new(alert).as_json
+          json[:artifacts] = json[:artifacts] || []
+          json[:tags] = json[:tags] || []
+          json
+        end
+      end
+
+      def count(artifact_data: nil, description: nil, source: nil, tag_name: nil, title: nil, from_at: nil, to_at: nil)
+        relation = build_relation(artifact_data: artifact_data, title: title, description: description, source: source, tag_name: tag_name, from_at: from_at, to_at: to_at)
+        relation.distinct("alerts.id").count
+      end
+
+      private
+
+      def build_relation(artifact_data: nil, title: nil, description: nil, source: nil, tag_name: nil, from_at: nil, to_at: nil)
+        relation = self
+        relation = joins(:tags) if tag_name
+        relation = joins(:artifacts) if artifact_data
+
+        relation = relation.where(artifacts: {data: artifact_data}) if artifact_data
+        relation = relation.where(tags: {name: tag_name}) if tag_name
+
+        relation = relation.where(source: source) if source
+        relation = relation.where(title: title) if title
+
+        relation = relation.filter(description: {like: "%#{description}%"}) if description
+
+        relation = relation.filter(created_at: {gte: from_at}) if from_at
+        relation = relation.filter(created_at: {lte: to_at}) if to_at
+
+        relation
+      end
+    end
   end
 end

data/lib/mihari/models/artifact.rb

@@ -6,7 +6,7 @@ class ArtifactValidator < ActiveModel::Validator
   def validate(record)
     return if record.data_type
 
-    record.errors[:data] << "#{record.data} is not supported"
+    record.errors.add :data, "#{record.data} is not supported"
   end
 end
 

data/lib/mihari/notifiers/exception_notifier.rb

@@ -19,7 +19,7 @@ module Mihari
       def notify(exception)
         notify_to_stdout exception
 
-        clean_message = exception.message.tr('`', "'")
+        clean_message = exception.message.tr("`", "'")
         attachments = to_attachments(exception, clean_message)
         notify_to_slack(text: clean_message, attachments: attachments) if @slack.valid?
       end
@@ -51,20 +51,20 @@ module Mihari
 
       def to_fields(clean_message, backtrace)
         fields = [
-          { title: "Exception", value: clean_message },
-          { title: "Hostname", value: hostname }
+          {title: "Exception", value: clean_message},
+          {title: "Hostname", value: hostname}
         ]
 
         if backtrace
           formatted_backtrace = format_backtrace(backtrace)
-          fields << { title: "Backtrace", value: formatted_backtrace }
+          fields << {title: "Backtrace", value: formatted_backtrace}
         end
         fields
       end
 
       def hostname
         Socket.gethostname
-      rescue StandardError => _e
+      rescue => _e
         "N/A"
       end
 

data/lib/mihari/serializers/alert.rb

@@ -4,7 +4,7 @@ require "active_model_serializers"
 
 module Mihari
   class AlertSerializer < ActiveModel::Serializer
-    attributes :title, :description, :source, :created_at
+    attributes :id, :title, :description, :source, :created_at
 
     has_many :artifacts
     has_many :tags, through: :taggings

data/lib/mihari/serializers/artifact.rb

@@ -4,6 +4,6 @@ require "active_model_serializers"
 
 module Mihari
   class ArtifactSerializer < ActiveModel::Serializer
-    attributes :data, :data_type
+    attributes :id, :data, :data_type
   end
 end

data/lib/mihari/serializers/tag.rb

@@ -4,6 +4,6 @@ require "active_model_serializers"
 
 module Mihari
   class TagSerializer < ActiveModel::Serializer
-    attributes :name
+    attributes :id, :name
   end
 end

data/lib/mihari/status.rb

@@ -3,9 +3,7 @@
 module Mihari
   class Status
     def check
-      statuses.map do |key, value|
-        [key, convert(**value)]
-      end.to_h
+      statuses.transform_values { |value| convert(**value) }
     end
 
     def self.check
@@ -14,16 +12,17 @@ module Mihari
 
     private
 
-    def convert(status:, message:)
+    def convert(is_configured:, values:, type:)
       {
-        status: status ? "OK" : "Bad",
-        message: message
+        is_configured: is_configured,
+        values: values,
+        type: type
       }
     end
 
     def statuses
       (Mihari.analyzers + Mihari.emitters).map do |klass|
-        name = klass.to_s.downcase.split("::").last.to_s
+        name = klass.to_s.split("::").last.to_s
 
         [name, build_status(klass)]
       end.to_h.compact
@@ -33,10 +32,11 @@ module Mihari
       is_analyzer = klass.ancestors.include?(Mihari::Analyzers::Base)
 
       instance = is_analyzer ? klass.new("dummy") : klass.new
-      status = instance.configured?
-      message = instance.configuration_status
+      is_configured = instance.configured?
+      values = instance.configuration_values
+      type = is_analyzer ? "Analyzer" : "Emitter"
 
-      message ? { status: status, message: message } : nil
+      values ? {is_configured: is_configured, values: values, type: type} : nil
     rescue ArgumentError => _e
       nil
     end