mihari 0.17.5 → 1.2.0

data/lib/mihari/config.rb

@@ -4,6 +4,60 @@ require "yaml"
 
 module Mihari
   class Config
+    attr_accessor :binaryedge_api_key
+    attr_accessor :censys_id
+    attr_accessor :censys_secret
+    attr_accessor :circl_passive_password
+    attr_accessor :circl_passive_username
+    attr_accessor :misp_api_endpoint
+    attr_accessor :misp_api_key
+    attr_accessor :onyphe_api_key
+    attr_accessor :otx_api_key
+    attr_accessor :passivetotal_api_key
+    attr_accessor :passivetotal_username
+    attr_accessor :pulsedive_api_key
+    attr_accessor :securitytrails_api_key
+    attr_accessor :shodan_api_key
+    attr_accessor :slack_channel
+    attr_accessor :slack_webhook_url
+    attr_accessor :thehive_api_endpoint
+    attr_accessor :thehive_api_key
+    attr_accessor :virustotal_api_key
+    attr_accessor :zoomeye_password
+    attr_accessor :zoomeye_username
+
+    attr_accessor :database
+
+    def initialize
+      load_from_env
+    end
+
+    def load_from_env
+      @binaryedge_api_key = ENV["BINARYEDGE_API_KEY"]
+      @censys_id = ENV["CENSYS_ID"]
+      @censys_secret = ENV["CENSYS_SECRET"]
+      @circl_passive_password = ENV["CIRCL_PASSIVE_PASSWORD"]
+      @circl_passive_username = ENV["CIRCL_PASSIVE_USERNAME"]
+      @misp_api_endpoint = ENV["MISP_API_ENDPOINT"]
+      @misp_api_key = ENV["MISP_API_KEY"]
+      @onyphe_api_key = ENV["ONYPHE_API_KEY"]
+      @otx_api_key = ENV["OTX_API_KEY"]
+      @passivetotal_api_key = ENV["PASSIVETOTAL_API_KEY"]
+      @passivetotal_username = ENV["PASSIVETOTAL_USERNAME"]
+      @pulsedive_api_key = ENV["PULSEDIVE_API_KEY"]
+      @securitytrails_api_key = ENV["SECURITYTRAILS_API_KEY"]
+      @shodan_api_key = ENV["SHODAN_API_KEY"]
+      @slack_channel = ENV["SLACK_CHANNEL"]
+      @slack_webhook_url = ENV["SLACK_WEBHOOK_URL"]
+      @thehive_api_endpoint = ENV["THEHIVE_API_ENDPOINT"]
+      @thehive_api_key = ENV["THEHIVE_API_KEY"]
+      @virustotal_api_key = ENV["VIRUSTOTAL_API_KEY"]
+      @zoomeye_password = ENV["ZOOMEYE_PASSWORD"]
+      @zoomeye_username = ENV["ZOOMEYE_USERNAME"]
+
+      @database = ENV["DATABASE"] || "mihari.db"
+    end
+
     class << self
       def load_from_yaml(path)
         raise ArgumentError, "#{path} does not exist." unless File.exist?(path)
@@ -15,10 +69,24 @@ module Mihari
           return
         end
 
-        yaml.each do |key, value|
-          ENV[key.upcase] = value
+        Mihari.configure do |config|
+          yaml.each do |key, value|
+            config.send("#{key.downcase}=".to_sym, value)
+          end
         end
       end
     end
   end
+
+  class << self
+    def config
+      @config ||= Config.new
+    end
+
+    attr_writer :config
+
+    def configure
+      yield config
+    end
+  end
 end

data/lib/mihari/configurable.rb

@@ -3,7 +3,7 @@
 module Mihari
   module Configurable
     def configured?
-      config_keys.all? { |key| ENV.key? key }
+      config_keys.all? { |key| Mihari.config.send(key) }
     end
 
     def configuration_status

data/lib/mihari/database.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require "active_record"
+
+class InitialSchema < ActiveRecord::Migration[6.0]
+  def change
+    create_table :tags, if_not_exists: true do |t|
+      t.string :name, null: false
+    end
+
+    create_table :alerts, if_not_exists: true do |t|
+      t.string :title, null: false
+      t.string :description, null: true
+      t.string :source, null: false
+      t.timestamps
+    end
+
+    create_table :artifacts, if_not_exists: true do |t|
+      t.string :data, null: false
+      t.string :data_type, null: false
+      t.belongs_to :alert, foreign_key: true
+      t.timestamps
+    end
+
+    create_table :taggings, if_not_exists: true do |t|
+      t.integer :tag_id
+      t.integer :alert_id
+    end
+
+    add_index :taggings, :tag_id, if_not_exists: true
+    add_index :taggings, [:tag_id, :alert_id], unique: true, if_not_exists: true
+  end
+end
+
+def adapter
+  return "postgresql" if Mihari.config.database.start_with?("postgresql://", "postgres://")
+
+  "sqlite3"
+end
+
+module Mihari
+  class Database
+    class << self
+      def connect
+        case adapter
+        when "postgresql"
+          ActiveRecord::Base.establish_connection(Mihari.config.database)
+        else
+          ActiveRecord::Base.establish_connection(
+            adapter: adapter,
+            database: Mihari.config.database
+          )
+        end
+
+        ActiveRecord::Migration.verbose = false
+        InitialSchema.migrate(:up)
+      rescue StandardError
+        # Do nothing
+      end
+
+      def destroy!
+        InitialSchema.migrate(:down) if ActiveRecord::Base.connected?
+      end
+    end
+  end
+end
+
+Mihari::Database.connect

data/lib/mihari/emitters/base.rb

@@ -16,7 +16,7 @@ module Mihari
       end
 
       def run(**params)
-        retry_on_error { emit(params) }
+        retry_on_error { emit(**params) }
       end
 
       def emit(*)

data/lib/mihari/emitters/database.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Emitters
+    class Database < Base
+      def valid?
+        true
+      end
+
+      def emit(title:, description:, artifacts:, source:, tags: [])
+        return if artifacts.empty?
+
+        tags = tags.map { |name| Tag.find_or_create_by(name: name) }.compact.uniq
+        taggings = tags.map { |tag| Tagging.new(tag_id: tag.id) }
+
+        alert = Alert.new(
+          title: title,
+          description: description,
+          artifacts: artifacts,
+          source: source,
+          taggings: taggings
+        )
+
+        alert.save
+        alert
+      end
+    end
+  end
+end

data/lib/mihari/emitters/misp.rb

@@ -6,6 +6,13 @@ require "net/ping"
 module Mihari
   module Emitters
     class MISP < Base
+      def initialize
+        ::MISP.configure do |config|
+          config.api_endpoint = Mihari.config.misp_api_endpoint
+          config.api_key = Mihari.config.misp_api_key
+        end
+      end
+
       # @return [true, false]
       def valid?
         api_endpoint? && api_key? && ping?
@@ -28,7 +35,7 @@ module Mihari
       private
 
       def config_keys
-        %w(MISP_API_ENDPOINT MISP_API_KEY)
+        %w(misp_api_endpoint misp_api_key)
       end
 
       def build_attribute(artifact)

data/lib/mihari/emitters/slack.rb

@@ -4,6 +4,8 @@ require "slack-notifier"
 require "digest/sha2"
 require "mem"
 
+require "mihari/slack_monkeypatch"
+
 module Mihari
   module Emitters
     class Attachment
@@ -123,7 +125,7 @@ module Mihari
         ].join("\n")
       end
 
-      def emit(title:, description:, artifacts:, tags: [])
+      def emit(title:, description:, artifacts:, tags: [], **_options)
         return if artifacts.empty?
 
         attachments = to_attachments(artifacts)
@@ -135,7 +137,7 @@ module Mihari
       private
 
       def config_keys
-        %w(SLACK_WEBHOOK_URL)
+        %w(slack_webhook_url)
       end
     end
   end

data/lib/mihari/emitters/stdout.rb

@@ -9,11 +9,12 @@ module Mihari
         true
       end
 
-      def emit(title:, description:, artifacts:, tags:)
+      def emit(title:, description:, artifacts:, source:, tags:)
         h = {
           title: title,
           description: description,
           artifacts: artifacts.map(&:data),
+          source: source,
           tags: tags
         }
         puts JSON.pretty_generate(h)

data/lib/mihari/emitters/the_hive.rb

@@ -1,42 +1,56 @@
 # frozen_string_literal: true
 
+require "hachi"
+require "net/ping"
+
 module Mihari
   module Emitters
     class TheHive < Base
       # @return [true, false]
       def valid?
-        the_hive.valid?
+        api_endpont? && api_key? && ping?
       end
 
-      def emit(title:, description:, artifacts:, tags: [])
+      def emit(title:, description:, artifacts:, tags: [], **_options)
         return if artifacts.empty?
 
-        the_hive.alert.create(
+        api.alert.create(
           title: title,
           description: description,
-          artifacts: artifacts.map(&:to_h),
-          tags: tags
+          artifacts: artifacts.map { |artifact| { data: artifact.data, data_type: artifact.data_type, message: description } },
+          tags: tags,
+          type: "external",
+          source: "mihari"
         )
-
-        save_as_cache artifacts.map(&:data)
       end
 
       private
 
       def config_keys
-        %w(THEHIVE_API_ENDPOINT THEHIVE_API_KEY)
+        %w(thehive_api_endpoint thehive_api_key)
       end
 
-      def the_hive
-        @the_hive ||= Mihari::TheHive.new
+      def api
+        @api ||= Hachi::API.new(api_endpoint: Mihari.config.thehive_api_endpoint, api_key: Mihari.config.thehive_api_key)
       end
 
-      def cache
-        @cache ||= Cache.new
+      # @return [true, false]
+      def api_endpont?
+        !Mihari.config.thehive_api_endpoint.nil?
       end
 
-      def save_as_cache(data)
-        cache.save data
+      # @return [true, false]
+      def api_key?
+        !Mihari.config.thehive_api_key.nil?
+      end
+
+      def ping?
+        base_url = Mihari.config.thehive_api_endpoint
+        base_url = base_url.end_with?("/") ? base_url[0..-2] : base_url
+        url = "#{base_url}/index.html"
+
+        http = Net::Ping::HTTP.new(url)
+        http.ping?
       end
     end
   end

data/lib/mihari/models/alert.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require "active_record"
+
+module Mihari
+  class Alert < ActiveRecord::Base
+    has_many :taggings, dependent: :destroy
+    has_many :artifacts, dependent: :destroy
+    has_many :tags, through: :taggings
+  end
+end

data/lib/mihari/models/artifact.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "active_record"
+
+class ArtifactValidator < ActiveModel::Validator
+  def validate(record)
+    return if record.data_type
+
+    record.errors[:data] << "#{record.data} is not supported"
+  end
+end
+
+module Mihari
+  class Artifact < ActiveRecord::Base
+    include ActiveModel::Validations
+    validates_with ArtifactValidator
+
+    def initialize(attributes)
+      super
+      self.data_type = TypeChecker.type(data)
+    end
+
+    def unique?
+      self.class.find_by(data: data).nil?
+    end
+  end
+end

data/lib/mihari/models/tag.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require "active_record"
+
+module Mihari
+  class Tag < ActiveRecord::Base
+    has_many :taggings, dependent: :destroy
+    has_many :tags, through: :taggings
+  end
+end

data/lib/mihari/models/tagging.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require "active_record"
+
+module Mihari
+  class Tagging < ActiveRecord::Base
+    belongs_to :alert
+    belongs_to :tag
+  end
+end

data/lib/mihari/notifiers/slack.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require "slack-notifier"
+require "mihari/slack_monkeypatch"
+
 module Mihari
   module Notifiers
     class Slack < Base
@@ -8,15 +11,15 @@ module Mihari
       DEFAULT_USERNAME = "mihari"
 
       def slack_channel
-        ENV.fetch SLACK_CHANNEL_KEY, "#general"
+        Mihari.config.slack_channel || "#general"
       end
 
       def slack_webhook_url
-        ENV.fetch SLACK_WEBHOOK_URL_KEY
+        Mihari.config.slack_webhook_url
       end
 
       def slack_webhook_url?
-        ENV.key? SLACK_WEBHOOK_URL_KEY
+        !Mihari.config.slack_webhook_url.nil?
       end
 
       def valid?
@@ -25,7 +28,7 @@ module Mihari
 
       def notify(text:, attachments: [], mrkdwn: true)
         notifier = ::Slack::Notifier.new(slack_webhook_url, channel: slack_channel, username: DEFAULT_USERNAME)
-        notifier.post(text: text, attachments: attachments, mrkdwn: true)
+        notifier.post(text: text, attachments: attachments, mrkdwn: mrkdwn)
       end
     end
   end

data/lib/mihari/serializers/alert.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "active_model_serializers"
+
+module Mihari
+  class AlertSerializer < ActiveModel::Serializer
+    attributes :title, :description, :source, :created_at
+
+    has_many :artifacts
+    has_many :tags, through: :taggings
+  end
+end