mihari 0.17.5 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +3 -0
  3. data/.rubocop.yml +155 -0
  4. data/.travis.yml +7 -1
  5. data/Gemfile +2 -0
  6. data/README.md +45 -73
  7. data/config/pre_commit.yml +3 -0
  8. data/docker/Dockerfile +1 -1
  9. data/lib/mihari.rb +13 -8
  10. data/lib/mihari/alert_viewer.rb +16 -34
  11. data/lib/mihari/analyzers/base.rb +7 -19
  12. data/lib/mihari/analyzers/basic.rb +3 -1
  13. data/lib/mihari/analyzers/binaryedge.rb +2 -2
  14. data/lib/mihari/analyzers/censys.rb +2 -2
  15. data/lib/mihari/analyzers/circl.rb +2 -2
  16. data/lib/mihari/analyzers/onyphe.rb +3 -3
  17. data/lib/mihari/analyzers/otx.rb +74 -0
  18. data/lib/mihari/analyzers/passive_dns.rb +2 -1
  19. data/lib/mihari/analyzers/passivetotal.rb +2 -2
  20. data/lib/mihari/analyzers/pulsedive.rb +2 -2
  21. data/lib/mihari/analyzers/securitytrails.rb +2 -2
  22. data/lib/mihari/analyzers/securitytrails_domain_feed.rb +2 -2
  23. data/lib/mihari/analyzers/shodan.rb +2 -2
  24. data/lib/mihari/analyzers/virustotal.rb +2 -2
  25. data/lib/mihari/analyzers/zoomeye.rb +2 -2
  26. data/lib/mihari/cli.rb +23 -4
  27. data/lib/mihari/config.rb +70 -2
  28. data/lib/mihari/configurable.rb +1 -1
  29. data/lib/mihari/database.rb +68 -0
  30. data/lib/mihari/emitters/base.rb +1 -1
  31. data/lib/mihari/emitters/database.rb +29 -0
  32. data/lib/mihari/emitters/misp.rb +8 -1
  33. data/lib/mihari/emitters/slack.rb +4 -2
  34. data/lib/mihari/emitters/stdout.rb +2 -1
  35. data/lib/mihari/emitters/the_hive.rb +28 -14
  36. data/lib/mihari/models/alert.rb +11 -0
  37. data/lib/mihari/models/artifact.rb +27 -0
  38. data/lib/mihari/models/tag.rb +10 -0
  39. data/lib/mihari/models/tagging.rb +10 -0
  40. data/lib/mihari/notifiers/slack.rb +7 -4
  41. data/lib/mihari/serializers/alert.rb +12 -0
  42. data/lib/mihari/serializers/artifact.rb +9 -0
  43. data/lib/mihari/serializers/tag.rb +9 -0
  44. data/lib/mihari/slack_monkeypatch.rb +16 -0
  45. data/lib/mihari/status.rb +1 -1
  46. data/lib/mihari/type_checker.rb +1 -1
  47. data/lib/mihari/version.rb +1 -1
  48. data/mihari.gemspec +13 -5
  49. metadata +149 -30
  50. data/lib/mihari/artifact.rb +0 -36
  51. data/lib/mihari/cache.rb +0 -35
  52. data/lib/mihari/the_hive.rb +0 -42
  53. data/lib/mihari/the_hive/alert.rb +0 -25
  54. data/lib/mihari/the_hive/artifact.rb +0 -33
  55. data/lib/mihari/the_hive/base.rb +0 -14
@@ -4,6 +4,60 @@ require "yaml"
4
4
 
5
5
  module Mihari
6
6
  class Config
7
+ attr_accessor :binaryedge_api_key
8
+ attr_accessor :censys_id
9
+ attr_accessor :censys_secret
10
+ attr_accessor :circl_passive_password
11
+ attr_accessor :circl_passive_username
12
+ attr_accessor :misp_api_endpoint
13
+ attr_accessor :misp_api_key
14
+ attr_accessor :onyphe_api_key
15
+ attr_accessor :otx_api_key
16
+ attr_accessor :passivetotal_api_key
17
+ attr_accessor :passivetotal_username
18
+ attr_accessor :pulsedive_api_key
19
+ attr_accessor :securitytrails_api_key
20
+ attr_accessor :shodan_api_key
21
+ attr_accessor :slack_channel
22
+ attr_accessor :slack_webhook_url
23
+ attr_accessor :thehive_api_endpoint
24
+ attr_accessor :thehive_api_key
25
+ attr_accessor :virustotal_api_key
26
+ attr_accessor :zoomeye_password
27
+ attr_accessor :zoomeye_username
28
+
29
+ attr_accessor :database
30
+
31
+ def initialize
32
+ load_from_env
33
+ end
34
+
35
+ def load_from_env
36
+ @binaryedge_api_key = ENV["BINARYEDGE_API_KEY"]
37
+ @censys_id = ENV["CENSYS_ID"]
38
+ @censys_secret = ENV["CENSYS_SECRET"]
39
+ @circl_passive_password = ENV["CIRCL_PASSIVE_PASSWORD"]
40
+ @circl_passive_username = ENV["CIRCL_PASSIVE_USERNAME"]
41
+ @misp_api_endpoint = ENV["MISP_API_ENDPOINT"]
42
+ @misp_api_key = ENV["MISP_API_KEY"]
43
+ @onyphe_api_key = ENV["ONYPHE_API_KEY"]
44
+ @otx_api_key = ENV["OTX_API_KEY"]
45
+ @passivetotal_api_key = ENV["PASSIVETOTAL_API_KEY"]
46
+ @passivetotal_username = ENV["PASSIVETOTAL_USERNAME"]
47
+ @pulsedive_api_key = ENV["PULSEDIVE_API_KEY"]
48
+ @securitytrails_api_key = ENV["SECURITYTRAILS_API_KEY"]
49
+ @shodan_api_key = ENV["SHODAN_API_KEY"]
50
+ @slack_channel = ENV["SLACK_CHANNEL"]
51
+ @slack_webhook_url = ENV["SLACK_WEBHOOK_URL"]
52
+ @thehive_api_endpoint = ENV["THEHIVE_API_ENDPOINT"]
53
+ @thehive_api_key = ENV["THEHIVE_API_KEY"]
54
+ @virustotal_api_key = ENV["VIRUSTOTAL_API_KEY"]
55
+ @zoomeye_password = ENV["ZOOMEYE_PASSWORD"]
56
+ @zoomeye_username = ENV["ZOOMEYE_USERNAME"]
57
+
58
+ @database = ENV["DATABASE"] || "mihari.db"
59
+ end
60
+
7
61
  class << self
8
62
  def load_from_yaml(path)
9
63
  raise ArgumentError, "#{path} does not exist." unless File.exist?(path)
@@ -15,10 +69,24 @@ module Mihari
15
69
  return
16
70
  end
17
71
 
18
- yaml.each do |key, value|
19
- ENV[key.upcase] = value
72
+ Mihari.configure do |config|
73
+ yaml.each do |key, value|
74
+ config.send("#{key.downcase}=".to_sym, value)
75
+ end
20
76
  end
21
77
  end
22
78
  end
23
79
  end
80
+
81
+ class << self
82
+ def config
83
+ @config ||= Config.new
84
+ end
85
+
86
+ attr_writer :config
87
+
88
+ def configure
89
+ yield config
90
+ end
91
+ end
24
92
  end
@@ -3,7 +3,7 @@
3
3
  module Mihari
4
4
  module Configurable
5
5
  def configured?
6
- config_keys.all? { |key| ENV.key? key }
6
+ config_keys.all? { |key| Mihari.config.send(key) }
7
7
  end
8
8
 
9
9
  def configuration_status
@@ -0,0 +1,68 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "active_record"
4
+
5
+ class InitialSchema < ActiveRecord::Migration[6.0]
6
+ def change
7
+ create_table :tags, if_not_exists: true do |t|
8
+ t.string :name, null: false
9
+ end
10
+
11
+ create_table :alerts, if_not_exists: true do |t|
12
+ t.string :title, null: false
13
+ t.string :description, null: true
14
+ t.string :source, null: false
15
+ t.timestamps
16
+ end
17
+
18
+ create_table :artifacts, if_not_exists: true do |t|
19
+ t.string :data, null: false
20
+ t.string :data_type, null: false
21
+ t.belongs_to :alert, foreign_key: true
22
+ t.timestamps
23
+ end
24
+
25
+ create_table :taggings, if_not_exists: true do |t|
26
+ t.integer :tag_id
27
+ t.integer :alert_id
28
+ end
29
+
30
+ add_index :taggings, :tag_id, if_not_exists: true
31
+ add_index :taggings, [:tag_id, :alert_id], unique: true, if_not_exists: true
32
+ end
33
+ end
34
+
35
+ def adapter
36
+ return "postgresql" if Mihari.config.database.start_with?("postgresql://", "postgres://")
37
+
38
+ "sqlite3"
39
+ end
40
+
41
+ module Mihari
42
+ class Database
43
+ class << self
44
+ def connect
45
+ case adapter
46
+ when "postgresql"
47
+ ActiveRecord::Base.establish_connection(Mihari.config.database)
48
+ else
49
+ ActiveRecord::Base.establish_connection(
50
+ adapter: adapter,
51
+ database: Mihari.config.database
52
+ )
53
+ end
54
+
55
+ ActiveRecord::Migration.verbose = false
56
+ InitialSchema.migrate(:up)
57
+ rescue StandardError
58
+ # Do nothing
59
+ end
60
+
61
+ def destroy!
62
+ InitialSchema.migrate(:down) if ActiveRecord::Base.connected?
63
+ end
64
+ end
65
+ end
66
+ end
67
+
68
+ Mihari::Database.connect
@@ -16,7 +16,7 @@ module Mihari
16
16
  end
17
17
 
18
18
  def run(**params)
19
- retry_on_error { emit(params) }
19
+ retry_on_error { emit(**params) }
20
20
  end
21
21
 
22
22
  def emit(*)
@@ -0,0 +1,29 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Mihari
4
+ module Emitters
5
+ class Database < Base
6
+ def valid?
7
+ true
8
+ end
9
+
10
+ def emit(title:, description:, artifacts:, source:, tags: [])
11
+ return if artifacts.empty?
12
+
13
+ tags = tags.map { |name| Tag.find_or_create_by(name: name) }.compact.uniq
14
+ taggings = tags.map { |tag| Tagging.new(tag_id: tag.id) }
15
+
16
+ alert = Alert.new(
17
+ title: title,
18
+ description: description,
19
+ artifacts: artifacts,
20
+ source: source,
21
+ taggings: taggings
22
+ )
23
+
24
+ alert.save
25
+ alert
26
+ end
27
+ end
28
+ end
29
+ end
@@ -6,6 +6,13 @@ require "net/ping"
6
6
  module Mihari
7
7
  module Emitters
8
8
  class MISP < Base
9
+ def initialize
10
+ ::MISP.configure do |config|
11
+ config.api_endpoint = Mihari.config.misp_api_endpoint
12
+ config.api_key = Mihari.config.misp_api_key
13
+ end
14
+ end
15
+
9
16
  # @return [true, false]
10
17
  def valid?
11
18
  api_endpoint? && api_key? && ping?
@@ -28,7 +35,7 @@ module Mihari
28
35
  private
29
36
 
30
37
  def config_keys
31
- %w(MISP_API_ENDPOINT MISP_API_KEY)
38
+ %w(misp_api_endpoint misp_api_key)
32
39
  end
33
40
 
34
41
  def build_attribute(artifact)
@@ -4,6 +4,8 @@ require "slack-notifier"
4
4
  require "digest/sha2"
5
5
  require "mem"
6
6
 
7
+ require "mihari/slack_monkeypatch"
8
+
7
9
  module Mihari
8
10
  module Emitters
9
11
  class Attachment
@@ -123,7 +125,7 @@ module Mihari
123
125
  ].join("\n")
124
126
  end
125
127
 
126
- def emit(title:, description:, artifacts:, tags: [])
128
+ def emit(title:, description:, artifacts:, tags: [], **_options)
127
129
  return if artifacts.empty?
128
130
 
129
131
  attachments = to_attachments(artifacts)
@@ -135,7 +137,7 @@ module Mihari
135
137
  private
136
138
 
137
139
  def config_keys
138
- %w(SLACK_WEBHOOK_URL)
140
+ %w(slack_webhook_url)
139
141
  end
140
142
  end
141
143
  end
@@ -9,11 +9,12 @@ module Mihari
9
9
  true
10
10
  end
11
11
 
12
- def emit(title:, description:, artifacts:, tags:)
12
+ def emit(title:, description:, artifacts:, source:, tags:)
13
13
  h = {
14
14
  title: title,
15
15
  description: description,
16
16
  artifacts: artifacts.map(&:data),
17
+ source: source,
17
18
  tags: tags
18
19
  }
19
20
  puts JSON.pretty_generate(h)
@@ -1,42 +1,56 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require "hachi"
4
+ require "net/ping"
5
+
3
6
  module Mihari
4
7
  module Emitters
5
8
  class TheHive < Base
6
9
  # @return [true, false]
7
10
  def valid?
8
- the_hive.valid?
11
+ api_endpont? && api_key? && ping?
9
12
  end
10
13
 
11
- def emit(title:, description:, artifacts:, tags: [])
14
+ def emit(title:, description:, artifacts:, tags: [], **_options)
12
15
  return if artifacts.empty?
13
16
 
14
- the_hive.alert.create(
17
+ api.alert.create(
15
18
  title: title,
16
19
  description: description,
17
- artifacts: artifacts.map(&:to_h),
18
- tags: tags
20
+ artifacts: artifacts.map { |artifact| { data: artifact.data, data_type: artifact.data_type, message: description } },
21
+ tags: tags,
22
+ type: "external",
23
+ source: "mihari"
19
24
  )
20
-
21
- save_as_cache artifacts.map(&:data)
22
25
  end
23
26
 
24
27
  private
25
28
 
26
29
  def config_keys
27
- %w(THEHIVE_API_ENDPOINT THEHIVE_API_KEY)
30
+ %w(thehive_api_endpoint thehive_api_key)
28
31
  end
29
32
 
30
- def the_hive
31
- @the_hive ||= Mihari::TheHive.new
33
+ def api
34
+ @api ||= Hachi::API.new(api_endpoint: Mihari.config.thehive_api_endpoint, api_key: Mihari.config.thehive_api_key)
32
35
  end
33
36
 
34
- def cache
35
- @cache ||= Cache.new
37
+ # @return [true, false]
38
+ def api_endpont?
39
+ !Mihari.config.thehive_api_endpoint.nil?
36
40
  end
37
41
 
38
- def save_as_cache(data)
39
- cache.save data
42
+ # @return [true, false]
43
+ def api_key?
44
+ !Mihari.config.thehive_api_key.nil?
45
+ end
46
+
47
+ def ping?
48
+ base_url = Mihari.config.thehive_api_endpoint
49
+ base_url = base_url.end_with?("/") ? base_url[0..-2] : base_url
50
+ url = "#{base_url}/index.html"
51
+
52
+ http = Net::Ping::HTTP.new(url)
53
+ http.ping?
40
54
  end
41
55
  end
42
56
  end
@@ -0,0 +1,11 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "active_record"
4
+
5
+ module Mihari
6
+ class Alert < ActiveRecord::Base
7
+ has_many :taggings, dependent: :destroy
8
+ has_many :artifacts, dependent: :destroy
9
+ has_many :tags, through: :taggings
10
+ end
11
+ end
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "active_record"
4
+
5
+ class ArtifactValidator < ActiveModel::Validator
6
+ def validate(record)
7
+ return if record.data_type
8
+
9
+ record.errors[:data] << "#{record.data} is not supported"
10
+ end
11
+ end
12
+
13
+ module Mihari
14
+ class Artifact < ActiveRecord::Base
15
+ include ActiveModel::Validations
16
+ validates_with ArtifactValidator
17
+
18
+ def initialize(attributes)
19
+ super
20
+ self.data_type = TypeChecker.type(data)
21
+ end
22
+
23
+ def unique?
24
+ self.class.find_by(data: data).nil?
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,10 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "active_record"
4
+
5
+ module Mihari
6
+ class Tag < ActiveRecord::Base
7
+ has_many :taggings, dependent: :destroy
8
+ has_many :tags, through: :taggings
9
+ end
10
+ end
@@ -0,0 +1,10 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "active_record"
4
+
5
+ module Mihari
6
+ class Tagging < ActiveRecord::Base
7
+ belongs_to :alert
8
+ belongs_to :tag
9
+ end
10
+ end
@@ -1,5 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require "slack-notifier"
4
+ require "mihari/slack_monkeypatch"
5
+
3
6
  module Mihari
4
7
  module Notifiers
5
8
  class Slack < Base
@@ -8,15 +11,15 @@ module Mihari
8
11
  DEFAULT_USERNAME = "mihari"
9
12
 
10
13
  def slack_channel
11
- ENV.fetch SLACK_CHANNEL_KEY, "#general"
14
+ Mihari.config.slack_channel || "#general"
12
15
  end
13
16
 
14
17
  def slack_webhook_url
15
- ENV.fetch SLACK_WEBHOOK_URL_KEY
18
+ Mihari.config.slack_webhook_url
16
19
  end
17
20
 
18
21
  def slack_webhook_url?
19
- ENV.key? SLACK_WEBHOOK_URL_KEY
22
+ !Mihari.config.slack_webhook_url.nil?
20
23
  end
21
24
 
22
25
  def valid?
@@ -25,7 +28,7 @@ module Mihari
25
28
 
26
29
  def notify(text:, attachments: [], mrkdwn: true)
27
30
  notifier = ::Slack::Notifier.new(slack_webhook_url, channel: slack_channel, username: DEFAULT_USERNAME)
28
- notifier.post(text: text, attachments: attachments, mrkdwn: true)
31
+ notifier.post(text: text, attachments: attachments, mrkdwn: mrkdwn)
29
32
  end
30
33
  end
31
34
  end
@@ -0,0 +1,12 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "active_model_serializers"
4
+
5
+ module Mihari
6
+ class AlertSerializer < ActiveModel::Serializer
7
+ attributes :title, :description, :source, :created_at
8
+
9
+ has_many :artifacts
10
+ has_many :tags, through: :taggings
11
+ end
12
+ end