mihari 0.17.5 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +3 -0
- data/.rubocop.yml +155 -0
- data/.travis.yml +7 -1
- data/Gemfile +2 -0
- data/README.md +45 -73
- data/config/pre_commit.yml +3 -0
- data/docker/Dockerfile +1 -1
- data/lib/mihari.rb +13 -8
- data/lib/mihari/alert_viewer.rb +16 -34
- data/lib/mihari/analyzers/base.rb +7 -19
- data/lib/mihari/analyzers/basic.rb +3 -1
- data/lib/mihari/analyzers/binaryedge.rb +2 -2
- data/lib/mihari/analyzers/censys.rb +2 -2
- data/lib/mihari/analyzers/circl.rb +2 -2
- data/lib/mihari/analyzers/onyphe.rb +3 -3
- data/lib/mihari/analyzers/otx.rb +74 -0
- data/lib/mihari/analyzers/passive_dns.rb +2 -1
- data/lib/mihari/analyzers/passivetotal.rb +2 -2
- data/lib/mihari/analyzers/pulsedive.rb +2 -2
- data/lib/mihari/analyzers/securitytrails.rb +2 -2
- data/lib/mihari/analyzers/securitytrails_domain_feed.rb +2 -2
- data/lib/mihari/analyzers/shodan.rb +2 -2
- data/lib/mihari/analyzers/virustotal.rb +2 -2
- data/lib/mihari/analyzers/zoomeye.rb +2 -2
- data/lib/mihari/cli.rb +23 -4
- data/lib/mihari/config.rb +70 -2
- data/lib/mihari/configurable.rb +1 -1
- data/lib/mihari/database.rb +68 -0
- data/lib/mihari/emitters/base.rb +1 -1
- data/lib/mihari/emitters/database.rb +29 -0
- data/lib/mihari/emitters/misp.rb +8 -1
- data/lib/mihari/emitters/slack.rb +4 -2
- data/lib/mihari/emitters/stdout.rb +2 -1
- data/lib/mihari/emitters/the_hive.rb +28 -14
- data/lib/mihari/models/alert.rb +11 -0
- data/lib/mihari/models/artifact.rb +27 -0
- data/lib/mihari/models/tag.rb +10 -0
- data/lib/mihari/models/tagging.rb +10 -0
- data/lib/mihari/notifiers/slack.rb +7 -4
- data/lib/mihari/serializers/alert.rb +12 -0
- data/lib/mihari/serializers/artifact.rb +9 -0
- data/lib/mihari/serializers/tag.rb +9 -0
- data/lib/mihari/slack_monkeypatch.rb +16 -0
- data/lib/mihari/status.rb +1 -1
- data/lib/mihari/type_checker.rb +1 -1
- data/lib/mihari/version.rb +1 -1
- data/mihari.gemspec +13 -5
- metadata +149 -30
- data/lib/mihari/artifact.rb +0 -36
- data/lib/mihari/cache.rb +0 -35
- data/lib/mihari/the_hive.rb +0 -42
- data/lib/mihari/the_hive/alert.rb +0 -25
- data/lib/mihari/the_hive/artifact.rb +0 -33
- data/lib/mihari/the_hive/base.rb +0 -14
data/lib/mihari/config.rb
CHANGED
@@ -4,6 +4,60 @@ require "yaml"
|
|
4
4
|
|
5
5
|
module Mihari
|
6
6
|
class Config
|
7
|
+
attr_accessor :binaryedge_api_key
|
8
|
+
attr_accessor :censys_id
|
9
|
+
attr_accessor :censys_secret
|
10
|
+
attr_accessor :circl_passive_password
|
11
|
+
attr_accessor :circl_passive_username
|
12
|
+
attr_accessor :misp_api_endpoint
|
13
|
+
attr_accessor :misp_api_key
|
14
|
+
attr_accessor :onyphe_api_key
|
15
|
+
attr_accessor :otx_api_key
|
16
|
+
attr_accessor :passivetotal_api_key
|
17
|
+
attr_accessor :passivetotal_username
|
18
|
+
attr_accessor :pulsedive_api_key
|
19
|
+
attr_accessor :securitytrails_api_key
|
20
|
+
attr_accessor :shodan_api_key
|
21
|
+
attr_accessor :slack_channel
|
22
|
+
attr_accessor :slack_webhook_url
|
23
|
+
attr_accessor :thehive_api_endpoint
|
24
|
+
attr_accessor :thehive_api_key
|
25
|
+
attr_accessor :virustotal_api_key
|
26
|
+
attr_accessor :zoomeye_password
|
27
|
+
attr_accessor :zoomeye_username
|
28
|
+
|
29
|
+
attr_accessor :database
|
30
|
+
|
31
|
+
def initialize
|
32
|
+
load_from_env
|
33
|
+
end
|
34
|
+
|
35
|
+
def load_from_env
|
36
|
+
@binaryedge_api_key = ENV["BINARYEDGE_API_KEY"]
|
37
|
+
@censys_id = ENV["CENSYS_ID"]
|
38
|
+
@censys_secret = ENV["CENSYS_SECRET"]
|
39
|
+
@circl_passive_password = ENV["CIRCL_PASSIVE_PASSWORD"]
|
40
|
+
@circl_passive_username = ENV["CIRCL_PASSIVE_USERNAME"]
|
41
|
+
@misp_api_endpoint = ENV["MISP_API_ENDPOINT"]
|
42
|
+
@misp_api_key = ENV["MISP_API_KEY"]
|
43
|
+
@onyphe_api_key = ENV["ONYPHE_API_KEY"]
|
44
|
+
@otx_api_key = ENV["OTX_API_KEY"]
|
45
|
+
@passivetotal_api_key = ENV["PASSIVETOTAL_API_KEY"]
|
46
|
+
@passivetotal_username = ENV["PASSIVETOTAL_USERNAME"]
|
47
|
+
@pulsedive_api_key = ENV["PULSEDIVE_API_KEY"]
|
48
|
+
@securitytrails_api_key = ENV["SECURITYTRAILS_API_KEY"]
|
49
|
+
@shodan_api_key = ENV["SHODAN_API_KEY"]
|
50
|
+
@slack_channel = ENV["SLACK_CHANNEL"]
|
51
|
+
@slack_webhook_url = ENV["SLACK_WEBHOOK_URL"]
|
52
|
+
@thehive_api_endpoint = ENV["THEHIVE_API_ENDPOINT"]
|
53
|
+
@thehive_api_key = ENV["THEHIVE_API_KEY"]
|
54
|
+
@virustotal_api_key = ENV["VIRUSTOTAL_API_KEY"]
|
55
|
+
@zoomeye_password = ENV["ZOOMEYE_PASSWORD"]
|
56
|
+
@zoomeye_username = ENV["ZOOMEYE_USERNAME"]
|
57
|
+
|
58
|
+
@database = ENV["DATABASE"] || "mihari.db"
|
59
|
+
end
|
60
|
+
|
7
61
|
class << self
|
8
62
|
def load_from_yaml(path)
|
9
63
|
raise ArgumentError, "#{path} does not exist." unless File.exist?(path)
|
@@ -15,10 +69,24 @@ module Mihari
|
|
15
69
|
return
|
16
70
|
end
|
17
71
|
|
18
|
-
|
19
|
-
|
72
|
+
Mihari.configure do |config|
|
73
|
+
yaml.each do |key, value|
|
74
|
+
config.send("#{key.downcase}=".to_sym, value)
|
75
|
+
end
|
20
76
|
end
|
21
77
|
end
|
22
78
|
end
|
23
79
|
end
|
80
|
+
|
81
|
+
class << self
|
82
|
+
def config
|
83
|
+
@config ||= Config.new
|
84
|
+
end
|
85
|
+
|
86
|
+
attr_writer :config
|
87
|
+
|
88
|
+
def configure
|
89
|
+
yield config
|
90
|
+
end
|
91
|
+
end
|
24
92
|
end
|
data/lib/mihari/configurable.rb
CHANGED
@@ -0,0 +1,68 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "active_record"
|
4
|
+
|
5
|
+
class InitialSchema < ActiveRecord::Migration[6.0]
|
6
|
+
def change
|
7
|
+
create_table :tags, if_not_exists: true do |t|
|
8
|
+
t.string :name, null: false
|
9
|
+
end
|
10
|
+
|
11
|
+
create_table :alerts, if_not_exists: true do |t|
|
12
|
+
t.string :title, null: false
|
13
|
+
t.string :description, null: true
|
14
|
+
t.string :source, null: false
|
15
|
+
t.timestamps
|
16
|
+
end
|
17
|
+
|
18
|
+
create_table :artifacts, if_not_exists: true do |t|
|
19
|
+
t.string :data, null: false
|
20
|
+
t.string :data_type, null: false
|
21
|
+
t.belongs_to :alert, foreign_key: true
|
22
|
+
t.timestamps
|
23
|
+
end
|
24
|
+
|
25
|
+
create_table :taggings, if_not_exists: true do |t|
|
26
|
+
t.integer :tag_id
|
27
|
+
t.integer :alert_id
|
28
|
+
end
|
29
|
+
|
30
|
+
add_index :taggings, :tag_id, if_not_exists: true
|
31
|
+
add_index :taggings, [:tag_id, :alert_id], unique: true, if_not_exists: true
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def adapter
|
36
|
+
return "postgresql" if Mihari.config.database.start_with?("postgresql://", "postgres://")
|
37
|
+
|
38
|
+
"sqlite3"
|
39
|
+
end
|
40
|
+
|
41
|
+
module Mihari
|
42
|
+
class Database
|
43
|
+
class << self
|
44
|
+
def connect
|
45
|
+
case adapter
|
46
|
+
when "postgresql"
|
47
|
+
ActiveRecord::Base.establish_connection(Mihari.config.database)
|
48
|
+
else
|
49
|
+
ActiveRecord::Base.establish_connection(
|
50
|
+
adapter: adapter,
|
51
|
+
database: Mihari.config.database
|
52
|
+
)
|
53
|
+
end
|
54
|
+
|
55
|
+
ActiveRecord::Migration.verbose = false
|
56
|
+
InitialSchema.migrate(:up)
|
57
|
+
rescue StandardError
|
58
|
+
# Do nothing
|
59
|
+
end
|
60
|
+
|
61
|
+
def destroy!
|
62
|
+
InitialSchema.migrate(:down) if ActiveRecord::Base.connected?
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
Mihari::Database.connect
|
data/lib/mihari/emitters/base.rb
CHANGED
@@ -0,0 +1,29 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Mihari
|
4
|
+
module Emitters
|
5
|
+
class Database < Base
|
6
|
+
def valid?
|
7
|
+
true
|
8
|
+
end
|
9
|
+
|
10
|
+
def emit(title:, description:, artifacts:, source:, tags: [])
|
11
|
+
return if artifacts.empty?
|
12
|
+
|
13
|
+
tags = tags.map { |name| Tag.find_or_create_by(name: name) }.compact.uniq
|
14
|
+
taggings = tags.map { |tag| Tagging.new(tag_id: tag.id) }
|
15
|
+
|
16
|
+
alert = Alert.new(
|
17
|
+
title: title,
|
18
|
+
description: description,
|
19
|
+
artifacts: artifacts,
|
20
|
+
source: source,
|
21
|
+
taggings: taggings
|
22
|
+
)
|
23
|
+
|
24
|
+
alert.save
|
25
|
+
alert
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
data/lib/mihari/emitters/misp.rb
CHANGED
@@ -6,6 +6,13 @@ require "net/ping"
|
|
6
6
|
module Mihari
|
7
7
|
module Emitters
|
8
8
|
class MISP < Base
|
9
|
+
def initialize
|
10
|
+
::MISP.configure do |config|
|
11
|
+
config.api_endpoint = Mihari.config.misp_api_endpoint
|
12
|
+
config.api_key = Mihari.config.misp_api_key
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
9
16
|
# @return [true, false]
|
10
17
|
def valid?
|
11
18
|
api_endpoint? && api_key? && ping?
|
@@ -28,7 +35,7 @@ module Mihari
|
|
28
35
|
private
|
29
36
|
|
30
37
|
def config_keys
|
31
|
-
%w(
|
38
|
+
%w(misp_api_endpoint misp_api_key)
|
32
39
|
end
|
33
40
|
|
34
41
|
def build_attribute(artifact)
|
@@ -4,6 +4,8 @@ require "slack-notifier"
|
|
4
4
|
require "digest/sha2"
|
5
5
|
require "mem"
|
6
6
|
|
7
|
+
require "mihari/slack_monkeypatch"
|
8
|
+
|
7
9
|
module Mihari
|
8
10
|
module Emitters
|
9
11
|
class Attachment
|
@@ -123,7 +125,7 @@ module Mihari
|
|
123
125
|
].join("\n")
|
124
126
|
end
|
125
127
|
|
126
|
-
def emit(title:, description:, artifacts:, tags: [])
|
128
|
+
def emit(title:, description:, artifacts:, tags: [], **_options)
|
127
129
|
return if artifacts.empty?
|
128
130
|
|
129
131
|
attachments = to_attachments(artifacts)
|
@@ -135,7 +137,7 @@ module Mihari
|
|
135
137
|
private
|
136
138
|
|
137
139
|
def config_keys
|
138
|
-
%w(
|
140
|
+
%w(slack_webhook_url)
|
139
141
|
end
|
140
142
|
end
|
141
143
|
end
|
@@ -9,11 +9,12 @@ module Mihari
|
|
9
9
|
true
|
10
10
|
end
|
11
11
|
|
12
|
-
def emit(title:, description:, artifacts:, tags:)
|
12
|
+
def emit(title:, description:, artifacts:, source:, tags:)
|
13
13
|
h = {
|
14
14
|
title: title,
|
15
15
|
description: description,
|
16
16
|
artifacts: artifacts.map(&:data),
|
17
|
+
source: source,
|
17
18
|
tags: tags
|
18
19
|
}
|
19
20
|
puts JSON.pretty_generate(h)
|
@@ -1,42 +1,56 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require "hachi"
|
4
|
+
require "net/ping"
|
5
|
+
|
3
6
|
module Mihari
|
4
7
|
module Emitters
|
5
8
|
class TheHive < Base
|
6
9
|
# @return [true, false]
|
7
10
|
def valid?
|
8
|
-
|
11
|
+
api_endpont? && api_key? && ping?
|
9
12
|
end
|
10
13
|
|
11
|
-
def emit(title:, description:, artifacts:, tags: [])
|
14
|
+
def emit(title:, description:, artifacts:, tags: [], **_options)
|
12
15
|
return if artifacts.empty?
|
13
16
|
|
14
|
-
|
17
|
+
api.alert.create(
|
15
18
|
title: title,
|
16
19
|
description: description,
|
17
|
-
artifacts: artifacts.map
|
18
|
-
tags: tags
|
20
|
+
artifacts: artifacts.map { |artifact| { data: artifact.data, data_type: artifact.data_type, message: description } },
|
21
|
+
tags: tags,
|
22
|
+
type: "external",
|
23
|
+
source: "mihari"
|
19
24
|
)
|
20
|
-
|
21
|
-
save_as_cache artifacts.map(&:data)
|
22
25
|
end
|
23
26
|
|
24
27
|
private
|
25
28
|
|
26
29
|
def config_keys
|
27
|
-
%w(
|
30
|
+
%w(thehive_api_endpoint thehive_api_key)
|
28
31
|
end
|
29
32
|
|
30
|
-
def
|
31
|
-
@
|
33
|
+
def api
|
34
|
+
@api ||= Hachi::API.new(api_endpoint: Mihari.config.thehive_api_endpoint, api_key: Mihari.config.thehive_api_key)
|
32
35
|
end
|
33
36
|
|
34
|
-
|
35
|
-
|
37
|
+
# @return [true, false]
|
38
|
+
def api_endpont?
|
39
|
+
!Mihari.config.thehive_api_endpoint.nil?
|
36
40
|
end
|
37
41
|
|
38
|
-
|
39
|
-
|
42
|
+
# @return [true, false]
|
43
|
+
def api_key?
|
44
|
+
!Mihari.config.thehive_api_key.nil?
|
45
|
+
end
|
46
|
+
|
47
|
+
def ping?
|
48
|
+
base_url = Mihari.config.thehive_api_endpoint
|
49
|
+
base_url = base_url.end_with?("/") ? base_url[0..-2] : base_url
|
50
|
+
url = "#{base_url}/index.html"
|
51
|
+
|
52
|
+
http = Net::Ping::HTTP.new(url)
|
53
|
+
http.ping?
|
40
54
|
end
|
41
55
|
end
|
42
56
|
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "active_record"
|
4
|
+
|
5
|
+
class ArtifactValidator < ActiveModel::Validator
|
6
|
+
def validate(record)
|
7
|
+
return if record.data_type
|
8
|
+
|
9
|
+
record.errors[:data] << "#{record.data} is not supported"
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
module Mihari
|
14
|
+
class Artifact < ActiveRecord::Base
|
15
|
+
include ActiveModel::Validations
|
16
|
+
validates_with ArtifactValidator
|
17
|
+
|
18
|
+
def initialize(attributes)
|
19
|
+
super
|
20
|
+
self.data_type = TypeChecker.type(data)
|
21
|
+
end
|
22
|
+
|
23
|
+
def unique?
|
24
|
+
self.class.find_by(data: data).nil?
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -1,5 +1,8 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require "slack-notifier"
|
4
|
+
require "mihari/slack_monkeypatch"
|
5
|
+
|
3
6
|
module Mihari
|
4
7
|
module Notifiers
|
5
8
|
class Slack < Base
|
@@ -8,15 +11,15 @@ module Mihari
|
|
8
11
|
DEFAULT_USERNAME = "mihari"
|
9
12
|
|
10
13
|
def slack_channel
|
11
|
-
|
14
|
+
Mihari.config.slack_channel || "#general"
|
12
15
|
end
|
13
16
|
|
14
17
|
def slack_webhook_url
|
15
|
-
|
18
|
+
Mihari.config.slack_webhook_url
|
16
19
|
end
|
17
20
|
|
18
21
|
def slack_webhook_url?
|
19
|
-
|
22
|
+
!Mihari.config.slack_webhook_url.nil?
|
20
23
|
end
|
21
24
|
|
22
25
|
def valid?
|
@@ -25,7 +28,7 @@ module Mihari
|
|
25
28
|
|
26
29
|
def notify(text:, attachments: [], mrkdwn: true)
|
27
30
|
notifier = ::Slack::Notifier.new(slack_webhook_url, channel: slack_channel, username: DEFAULT_USERNAME)
|
28
|
-
notifier.post(text: text, attachments: attachments, mrkdwn:
|
31
|
+
notifier.post(text: text, attachments: attachments, mrkdwn: mrkdwn)
|
29
32
|
end
|
30
33
|
end
|
31
34
|
end
|
@@ -0,0 +1,12 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "active_model_serializers"
|
4
|
+
|
5
|
+
module Mihari
|
6
|
+
class AlertSerializer < ActiveModel::Serializer
|
7
|
+
attributes :title, :description, :source, :created_at
|
8
|
+
|
9
|
+
has_many :artifacts
|
10
|
+
has_many :tags, through: :taggings
|
11
|
+
end
|
12
|
+
end
|