mihari 0.17.4 → 1.1.1

data/lib/mihari/analyzers/base.rb

@@ -23,6 +23,10 @@ module Mihari
         raise NotImplementedError, "You must implement #{self.class}##{__method__}"
       end
 
+      def source
+        self.class.to_s.split("::").last
+      end
+
       # @return [Array<String>]
       def tags
         []
@@ -37,7 +41,7 @@ module Mihari
       end
 
       def run_emitter(emitter)
-        emitter.run(title: title, description: description, artifacts: unique_artifacts, tags: tags)
+        emitter.run(title: title, description: description, artifacts: unique_artifacts, source: source, tags: tags)
       rescue StandardError => e
         puts "Emission by #{emitter.class} is failed: #{e}"
       end
@@ -48,32 +52,16 @@ module Mihari
 
       private
 
-      def the_hive
-        @the_hive ||= TheHive.new
-      end
-
-      def cache
-        @cache ||= Cache.new
-      end
-
       # @return [Array<Mihari::Artifact>]
       def normalized_artifacts
         @normalized_artifacts ||= artifacts.compact.uniq.sort.map do |artifact|
-          artifact.is_a?(Artifact) ? artifact : Artifact.new(artifact)
+          artifact.is_a?(Artifact) ? artifact : Artifact.new(data: artifact)
         end.select(&:valid?)
       end
 
-      def uncached_artifacts
-        @uncached_artifacts ||= normalized_artifacts.reject do |artifact|
-          cache.cached? artifact.data
-        end
-      end
-
       # @return [Array<Mihari::Artifact>]
       def unique_artifacts
-        return uncached_artifacts unless the_hive.valid?
-
-        @unique_artifacts ||= the_hive.artifact.find_non_existing_artifacts(uncached_artifacts)
+        @unique_artifacts ||= normalized_artifacts.select(&:unique?)
       end
 
       def set_unique_artifacts

data/lib/mihari/analyzers/basic.rb

@@ -6,12 +6,14 @@ module Mihari
       attr_accessor :title
       attr_reader :description
       attr_reader :artifacts
+      attr_reader :source
       attr_reader :tags
 
-      def initialize(title:, description:, artifacts:, tags: [])
+      def initialize(title:, description:, artifacts:, source:, tags: [])
         @title = title
         @description = description
         @artifacts = artifacts
+        @source = source
         @tags = tags
       end
     end

data/lib/mihari/analyzers/binaryedge.rb

@@ -26,7 +26,7 @@ module Mihari
         results.map do |result|
           events = result.dig("events") || []
           events.map do |event|
-            event.dig "origin", "ip"
+            event.dig "target", "ip"
           end.compact
         end.flatten.compact.uniq
       end
@@ -52,11 +52,11 @@ module Mihari
       end
 
       def config_keys
-        %w(BINARYEDGE_API_KEY)
+        %w(binaryedge_api_key)
       end
 
       def api
-        @api ||= ::BinaryEdge::API.new
+        @api ||= ::BinaryEdge::API.new(Mihari.config.binaryedge_api_key)
       end
     end
   end

data/lib/mihari/analyzers/censys.rb

@@ -86,11 +86,11 @@ module Mihari
       end
 
       def config_keys
-        %w(CENSYS_ID CENSYS_SECRET)
+        %w(censys_id censys_secret)
       end
 
       def api
-        @api ||= ::Censys::API.new
+        @api ||= ::Censys::API.new(Mihari.config.censys_id, Mihari.config.censys_secret)
       end
     end
   end

data/lib/mihari/analyzers/circl.rb

@@ -27,11 +27,11 @@ module Mihari
       private
 
       def config_keys
-        %w(CIRCL_PASSIVE_USERNAME CIRCL_PASSIVE_PASSWORD)
+        %w(circl_passive_password circl_passive_username)
       end
 
       def api
-        @api ||= ::PassiveCIRCL::API.new
+        @api ||= ::PassiveCIRCL::API.new(username: Mihari.config.circl_passive_username, password: Mihari.config.circl_passive_password)
       end
 
       def lookup

data/lib/mihari/analyzers/onyphe.rb

@@ -35,15 +35,15 @@ module Mihari
       PAGE_SIZE = 10
 
       def config_keys
-        %w(ONYPHE_API_KEY)
+        %w(onyphe_api_key)
       end
 
       def api
-        @api ||= ::Onyphe::API.new
+        @api ||= ::Onyphe::API.new(Mihari.config.onyphe_api_key)
       end
 
       def search_with_page(query, page: 1)
-        api.datascan(query, page: page)
+        api.simple.datascan(query, page: page)
       end
 
       def search

data/lib/mihari/analyzers/passivetotal.rb

@@ -30,11 +30,11 @@ module Mihari
       private
 
       def config_keys
-        %w(PASSIVETOTAL_USERNAME PASSIVETOTAL_API_KEY)
+        %w(passivetotal_username passivetotal_api_key)
       end
 
       def api
-        @api ||= ::PassiveTotal::API.new
+        @api ||= ::PassiveTotal::API.new(username: Mihari.config.passivetotal_username, api_key: Mihari.config.passivetotal_api_key)
       end
 
       def valid_type?

data/lib/mihari/analyzers/pulsedive.rb

@@ -30,11 +30,11 @@ module Mihari
       private
 
       def config_keys
-        %w(PULSEDIVE_API_KEY)
+        %w(pulsedive_api_key)
       end
 
       def api
-        @api ||= ::Pulsedive::API.new
+        @api ||= ::Pulsedive::API.new(Mihari.config.pulsedive_api_key)
       end
 
       def valid_type?

data/lib/mihari/analyzers/securitytrails.rb

@@ -30,11 +30,11 @@ module Mihari
       private
 
       def config_keys
-        %w(SECURITYTRAILS_API_KEY)
+        %w(securitytrails_api_key)
       end
 
       def api
-        @api ||= ::SecurityTrails::API.new
+        @api ||= ::SecurityTrails::API.new(Mihari.config.securitytrails_api_key)
       end
 
       def valid_type?

data/lib/mihari/analyzers/securitytrails_domain_feed.rb

@@ -32,11 +32,11 @@ module Mihari
       private
 
       def config_keys
-        %w(SECURITYTRAILS_API_KEY)
+        %w(securitytrails_api_key)
       end
 
       def api
-        @api ||= ::SecurityTrails::API.new
+        @api ||= ::SecurityTrails::API.new(Mihari.config.securitytrails_api_key)
       end
 
       def valid_type?

data/lib/mihari/analyzers/shodan.rb

@@ -36,11 +36,11 @@ module Mihari
       PAGE_SIZE = 100
 
       def config_keys
-        %w(SHODAN_API_KEY)
+        %w(shodan_api_key)
       end
 
       def api
-        @api ||= ::Shodan::API.new
+        @api ||= ::Shodan::API.new(key: Mihari.config.shodan_api_key)
       end
 
       def search_with_page(query, page: 1)

data/lib/mihari/analyzers/virustotal.rb

@@ -30,11 +30,11 @@ module Mihari
       private
 
       def config_keys
-        %w(VIRUSTOTAL_API_KEY)
+        %w(virustotal_api_key)
       end
 
       def api
-        @api = ::VirusTotal::API.new
+        @api = ::VirusTotal::API.new(key: Mihari.config.virustotal_api_key)
       end
 
       def valid_type?

data/lib/mihari/analyzers/zoomeye.rb

@@ -41,11 +41,11 @@ module Mihari
       end
 
       def config_keys
-        %w(ZOOMEYE_USERNAME ZOOMEYE_PASSWORD)
+        %w(zoomeye_password zoomeye_username)
       end
 
       def api
-        @api ||= ::ZoomEye::API.new
+        @api ||= ::ZoomEye::API.new(username: Mihari.config.zoomeye_username, password: Mihari.config.zoomeye_password)
       end
 
       def convert_responses(responses)

data/lib/mihari/cli.rb

@@ -242,17 +242,22 @@ module Mihari
         artifacts = json.dig("artifacts")
         tags = json.dig("tags") || []
 
-        basic = Analyzers::Basic.new(title: title, description: description, artifacts: artifacts, tags: tags)
+        basic = Analyzers::Basic.new(title: title, description: description, artifacts: artifacts, source: "json", tags: tags)
         basic.run
       end
     end
 
     desc "alerts", "Show the alerts on TheHive"
     method_option :limit, type: :string, default: "5", desc: "Number of alerts to show (or 'all' to show all the alerts)"
+    method_option :title, type: :string, desc: "Title to filter"
+    method_option :source, type: :string, desc: "Source to filter"
+    method_option :tag, type: :string, desc: "Tag to filter"
     def alerts
       with_error_handling do
-        viewer = AlertViewer.new(limit: options["limit"])
-        alerts = viewer.list
+        load_configuration
+
+        viewer = AlertViewer.new
+        alerts = viewer.list(limit: options["limit"], title: options["title"], source: options["source"], tag: options[:tag])
         puts JSON.pretty_generate(alerts)
       end
     end
@@ -261,6 +266,7 @@ module Mihari
     def status
       with_error_handling do
         load_configuration
+
         puts JSON.pretty_generate(Status.check)
       end
     end
@@ -286,7 +292,10 @@ module Mihari
 
       def load_configuration
         config = options["config"]
-        Config.load_from_yaml(config) if config
+        return unless config
+
+        Config.load_from_yaml(config)
+        Database.connect
       end
 
       def run_analyzer(analyzer_class, query:, options:)

data/lib/mihari/config.rb

@@ -4,6 +4,58 @@ require "yaml"
 
 module Mihari
   class Config
+    attr_accessor :binaryedge_api_key
+    attr_accessor :censys_id
+    attr_accessor :censys_secret
+    attr_accessor :circl_passive_password
+    attr_accessor :circl_passive_username
+    attr_accessor :misp_api_endpoint
+    attr_accessor :misp_api_key
+    attr_accessor :onyphe_api_key
+    attr_accessor :passivetotal_api_key
+    attr_accessor :passivetotal_username
+    attr_accessor :pulsedive_api_key
+    attr_accessor :securitytrails_api_key
+    attr_accessor :shodan_api_key
+    attr_accessor :slack_channel
+    attr_accessor :slack_webhook_url
+    attr_accessor :thehive_api_endpoint
+    attr_accessor :thehive_api_key
+    attr_accessor :virustotal_api_key
+    attr_accessor :zoomeye_password
+    attr_accessor :zoomeye_username
+
+    attr_accessor :database
+
+    def initialize
+      load_from_env
+    end
+
+    def load_from_env
+      @binaryedge_api_key = ENV["BINARYEDGE_API_KEY"]
+      @censys_id = ENV["CENSYS_ID"]
+      @censys_secret = ENV["CENSYS_SECRET"]
+      @circl_passive_password = ENV["CIRCL_PASSIVE_PASSWORD"]
+      @circl_passive_username = ENV["CIRCL_PASSIVE_USERNAME"]
+      @misp_api_endpoint = ENV["MISP_API_ENDPOINT"]
+      @misp_api_key = ENV["MISP_API_KEY"]
+      @onyphe_api_key = ENV["ONYPHE_API_KEY"]
+      @passivetotal_api_key = ENV["PASSIVETOTAL_API_KEY"]
+      @passivetotal_username = ENV["PASSIVETOTAL_USERNAME"]
+      @pulsedive_api_key = ENV["PULSEDIVE_API_KEY"]
+      @securitytrails_api_key = ENV["SECURITYTRAILS_API_KEY"]
+      @shodan_api_key = ENV["SHODAN_API_KEY"]
+      @slack_channel = ENV["SLACK_CHANNEL"]
+      @slack_webhook_url = ENV["SLACK_WEBHOOK_URL"]
+      @thehive_api_endpoint = ENV["THEHIVE_API_ENDPOINT"]
+      @thehive_api_key = ENV["THEHIVE_API_KEY"]
+      @virustotal_api_key = ENV["VIRUSTOTAL_API_KEY"]
+      @zoomeye_password = ENV["ZOOMEYE_PASSWORD"]
+      @zoomeye_username = ENV["ZOOMEYE_USERNAME"]
+
+      @database = ENV["DATABASE"] || "mihari.db"
+    end
+
     class << self
       def load_from_yaml(path)
         raise ArgumentError, "#{path} does not exist." unless File.exist?(path)
@@ -15,10 +67,24 @@ module Mihari
           return
         end
 
-        yaml.each do |key, value|
-          ENV[key.upcase] = value
+        Mihari.configure do |config|
+          yaml.each do |key, value|
+            config.send("#{key.downcase}=".to_sym, value)
+          end
         end
       end
     end
   end
+
+  class << self
+    def config
+      @config ||= Config.new
+    end
+
+    attr_writer :config
+
+    def configure
+      yield config
+    end
+  end
 end

data/lib/mihari/configurable.rb

@@ -3,7 +3,7 @@
 module Mihari
   module Configurable
     def configured?
-      config_keys.all? { |key| ENV.key? key }
+      config_keys.all? { |key| Mihari.config.send(key) }
     end
 
     def configuration_status

data/lib/mihari/database.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require "active_record"
+
+class InitialSchema < ActiveRecord::Migration[6.0]
+  def change
+    create_table :tags, if_not_exists: true do |t|
+      t.string :name, null: false
+    end
+
+    create_table :alerts, if_not_exists: true do |t|
+      t.string :title, null: false
+      t.string :description, null: true
+      t.string :source, null: false
+      t.timestamps
+    end
+
+    create_table :artifacts, if_not_exists: true do |t|
+      t.string :data, null: false
+      t.string :data_type, null: false
+      t.belongs_to :alert, foreign_key: true
+      t.timestamps
+    end
+
+    create_table :taggings, if_not_exists: true do |t|
+      t.integer :tag_id
+      t.integer :alert_id
+    end
+
+    add_index :taggings, :tag_id, if_not_exists: true
+    add_index :taggings, [:tag_id, :alert_id], unique: true, if_not_exists: true
+  end
+end
+
+def adapter
+  return "postgresql" if Mihari.config.database.start_with?("postgresql://", "postgres://")
+
+  "sqlite3"
+end
+
+module Mihari
+  class Database
+    class << self
+      def connect
+        case adapter
+        when "postgresql"
+          ActiveRecord::Base.establish_connection(Mihari.config.database)
+        else
+          ActiveRecord::Base.establish_connection(
+            adapter: adapter,
+            database: Mihari.config.database
+          )
+        end
+
+        ActiveRecord::Migration.verbose = false
+        InitialSchema.migrate(:up)
+      rescue StandardError
+        # Do nothing
+      end
+
+      def destroy!
+        InitialSchema.migrate(:down) if ActiveRecord::Base.connected?
+      end
+    end
+  end
+end
+
+Mihari::Database.connect