mihari 2.2.0 → 3.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (119) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +7 -0
  3. data/.overcommit.yml +12 -0
  4. data/README.md +7 -9
  5. data/exe/mihari +1 -1
  6. data/images/tines.png +0 -0
  7. data/lib/mihari.rb +89 -15
  8. data/lib/mihari/analyzers/base.rb +49 -8
  9. data/lib/mihari/analyzers/basic.rb +1 -2
  10. data/lib/mihari/analyzers/binaryedge.rb +7 -13
  11. data/lib/mihari/analyzers/censys.rb +26 -63
  12. data/lib/mihari/analyzers/circl.rb +20 -17
  13. data/lib/mihari/analyzers/crtsh.rb +6 -13
  14. data/lib/mihari/analyzers/dnpedia.rb +6 -12
  15. data/lib/mihari/analyzers/dnstwister.rb +13 -10
  16. data/lib/mihari/analyzers/onyphe.rb +6 -12
  17. data/lib/mihari/analyzers/otx.rb +22 -19
  18. data/lib/mihari/analyzers/passivetotal.rb +22 -21
  19. data/lib/mihari/analyzers/pulsedive.rb +16 -13
  20. data/lib/mihari/analyzers/rule.rb +99 -0
  21. data/lib/mihari/analyzers/securitytrails.rb +22 -19
  22. data/lib/mihari/analyzers/shodan.rb +7 -13
  23. data/lib/mihari/analyzers/spyse.rb +12 -19
  24. data/lib/mihari/analyzers/urlscan.rb +20 -30
  25. data/lib/mihari/analyzers/virustotal.rb +25 -22
  26. data/lib/mihari/analyzers/zoomeye.rb +16 -22
  27. data/lib/mihari/cli/analyzer.rb +44 -0
  28. data/lib/mihari/cli/base.rb +27 -0
  29. data/lib/mihari/cli/init.rb +13 -0
  30. data/lib/mihari/cli/main.rb +30 -0
  31. data/lib/mihari/cli/mixins/utils.rb +88 -0
  32. data/lib/mihari/cli/validator.rb +11 -0
  33. data/lib/mihari/commands/binaryedge.rb +1 -1
  34. data/lib/mihari/commands/censys.rb +1 -1
  35. data/lib/mihari/commands/circl.rb +2 -2
  36. data/lib/mihari/commands/crtsh.rb +1 -1
  37. data/lib/mihari/commands/dnpedia.rb +1 -1
  38. data/lib/mihari/commands/dnstwister.rb +2 -2
  39. data/lib/mihari/commands/init.rb +46 -0
  40. data/lib/mihari/commands/json.rb +1 -1
  41. data/lib/mihari/commands/onyphe.rb +1 -1
  42. data/lib/mihari/commands/otx.rb +2 -2
  43. data/lib/mihari/commands/passivetotal.rb +2 -2
  44. data/lib/mihari/commands/pulsedive.rb +2 -2
  45. data/lib/mihari/commands/search.rb +77 -0
  46. data/lib/mihari/commands/securitytrails.rb +2 -2
  47. data/lib/mihari/commands/shodan.rb +1 -1
  48. data/lib/mihari/commands/spyse.rb +1 -1
  49. data/lib/mihari/commands/urlscan.rb +2 -4
  50. data/lib/mihari/commands/validator.rb +38 -0
  51. data/lib/mihari/commands/virustotal.rb +2 -2
  52. data/lib/mihari/commands/zoomeye.rb +1 -1
  53. data/lib/mihari/constraints.rb +5 -0
  54. data/lib/mihari/database.rb +13 -2
  55. data/lib/mihari/emitters/base.rb +2 -2
  56. data/lib/mihari/emitters/database.rb +1 -1
  57. data/lib/mihari/emitters/misp.rb +1 -1
  58. data/lib/mihari/emitters/slack.rb +5 -9
  59. data/lib/mihari/emitters/the_hive.rb +1 -1
  60. data/lib/mihari/emitters/webhook.rb +53 -0
  61. data/lib/mihari/mixins/configurable.rb +38 -0
  62. data/lib/mihari/mixins/configuration.rb +85 -0
  63. data/lib/mihari/mixins/hash.rb +20 -0
  64. data/lib/mihari/mixins/refang.rb +21 -0
  65. data/lib/mihari/mixins/retriable.rb +27 -0
  66. data/lib/mihari/mixins/rule.rb +79 -0
  67. data/lib/mihari/models/alert.rb +28 -1
  68. data/lib/mihari/models/artifact.rb +10 -0
  69. data/lib/mihari/notifiers/base.rb +9 -1
  70. data/lib/mihari/notifiers/exception_notifier.rb +50 -0
  71. data/lib/mihari/notifiers/slack.rb +29 -1
  72. data/lib/mihari/schemas/configuration.rb +42 -0
  73. data/lib/mihari/schemas/macros.rb +17 -0
  74. data/lib/mihari/schemas/rule.rb +72 -0
  75. data/lib/mihari/serializers/artifact.rb +1 -1
  76. data/lib/mihari/status.rb +14 -0
  77. data/lib/mihari/templates/rule.yml.erb +19 -0
  78. data/lib/mihari/type_checker.rb +8 -3
  79. data/lib/mihari/version.rb +1 -1
  80. data/lib/mihari/web/app.rb +2 -0
  81. data/lib/mihari/web/controllers/alerts_controller.rb +12 -3
  82. data/lib/mihari/web/controllers/artifacts_controller.rb +1 -3
  83. data/lib/mihari/web/controllers/base_controller.rb +22 -0
  84. data/lib/mihari/web/controllers/command_controller.rb +3 -4
  85. data/lib/mihari/web/controllers/config_controller.rb +1 -3
  86. data/lib/mihari/web/controllers/sources_controller.rb +1 -3
  87. data/lib/mihari/web/controllers/tags_controller.rb +1 -3
  88. data/lib/mihari/web/helpers/json.rb +2 -0
  89. data/lib/mihari/web/public/index.html +1 -21
  90. data/lib/mihari/web/public/redoc-static.html +2 -2
  91. data/lib/mihari/web/public/static/js/app.ab213f7c.js +12 -0
  92. data/lib/mihari/web/public/static/js/app.ab213f7c.js.map +1 -0
  93. data/lib/mihari/web/public/static/js/{app.bcc595df.js → app.cccddb2b.js} +2 -2
  94. data/lib/mihari/web/public/static/js/app.cccddb2b.js.map +1 -0
  95. data/mihari.gemspec +20 -10
  96. metadata +180 -63
  97. data/.rubocop.yml +0 -161
  98. data/lib/mihari/analyzers/free_text.rb +0 -48
  99. data/lib/mihari/analyzers/http_hash.rb +0 -100
  100. data/lib/mihari/analyzers/passive_dns.rb +0 -59
  101. data/lib/mihari/analyzers/passive_ssl.rb +0 -55
  102. data/lib/mihari/analyzers/reverse_whois.rb +0 -55
  103. data/lib/mihari/analyzers/securitytrails_domain_feed.rb +0 -59
  104. data/lib/mihari/analyzers/ssh_fingerprint.rb +0 -58
  105. data/lib/mihari/cli.rb +0 -124
  106. data/lib/mihari/commands/config.rb +0 -27
  107. data/lib/mihari/commands/free_text.rb +0 -21
  108. data/lib/mihari/commands/http_hash.rb +0 -25
  109. data/lib/mihari/commands/passive_dns.rb +0 -21
  110. data/lib/mihari/commands/passive_ssl.rb +0 -21
  111. data/lib/mihari/commands/reverse_whois.rb +0 -21
  112. data/lib/mihari/commands/securitytrails_domain_feed.rb +0 -23
  113. data/lib/mihari/commands/ssh_fingerprint.rb +0 -21
  114. data/lib/mihari/config.rb +0 -84
  115. data/lib/mihari/configurable.rb +0 -21
  116. data/lib/mihari/html.rb +0 -43
  117. data/lib/mihari/retriable.rb +0 -17
  118. data/lib/mihari/slack_monkeypatch.rb +0 -16
  119. data/lib/mihari/web/public/static/js/app.bcc595df.js.map +0 -1
@@ -5,13 +5,13 @@ module Mihari
5
5
  module SecurityTrails
6
6
  def self.included(thor)
7
7
  thor.class_eval do
8
- desc "securitytrails [IP|DOMAIN|EMAIL]", "SecurityTrails lookup by an ip, domain or email"
8
+ desc "securitytrails [IP|DOMAIN|EMAIL]", "SecurityTrails search by an ip, domain or email"
9
9
  method_option :title, type: :string, desc: "title"
10
10
  method_option :description, type: :string, desc: "description"
11
11
  method_option :tags, type: :array, desc: "tags"
12
12
  def securitytrails(indiactor)
13
13
  with_error_handling do
14
- run_analyzer Analyzers::SecurityTrails, query: refang(indiactor), options: options
14
+ run_analyzer Analyzers::SecurityTrails, query: indiactor, options: options
15
15
  end
16
16
  end
17
17
  map "st" => :securitytrails
@@ -5,7 +5,7 @@ module Mihari
5
5
  module Shodan
6
6
  def self.included(thor)
7
7
  thor.class_eval do
8
- desc "shodan [QUERY]", "Shodan host search by a query"
8
+ desc "shodan [QUERY]", "Shodan host search"
9
9
  method_option :title, type: :string, desc: "title"
10
10
  method_option :description, type: :string, desc: "description"
11
11
  method_option :tags, type: :array, desc: "tags"
@@ -5,7 +5,7 @@ module Mihari
5
5
  module Spyse
6
6
  def self.included(thor)
7
7
  thor.class_eval do
8
- desc "spyse [QUERY]", "Spyse search by a query"
8
+ desc "spyse [QUERY]", "Spyse search"
9
9
  method_option :title, type: :string, desc: "title"
10
10
  method_option :description, type: :string, desc: "description"
11
11
  method_option :tags, type: :array, desc: "tags"
@@ -5,13 +5,11 @@ module Mihari
5
5
  module Urlscan
6
6
  def self.included(thor)
7
7
  thor.class_eval do
8
- desc "urlscan [QUERY]", "urlscan search by a given query"
8
+ desc "urlscan [QUERY]", "urlscan search"
9
9
  method_option :title, type: :string, desc: "title"
10
10
  method_option :description, type: :string, desc: "description"
11
11
  method_option :tags, type: :array, desc: "tags"
12
- method_option :filter, type: :string, desc: "filter for urlscan pro search"
13
- method_option :target_type, type: :string, default: "url", desc: "target type to fetch from lookup results (target type should be 'url', 'domain' or 'ip')"
14
- method_option :use_pro, type: :boolean, default: false, desc: "use pro search API or not"
12
+ method_option :target_type, type: :string, default: "url", desc: "target type to fetch from search results (target type should be 'url', 'domain' or 'ip')"
15
13
  method_option :use_similarity, type: :boolean, default: false, desc: "use similarity API or not"
16
14
  def urlscan(query)
17
15
  with_error_handling do
@@ -0,0 +1,38 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Mihari
4
+ module Commands
5
+ module Validator
6
+ include Mixins::Rule
7
+ include Mixins::Configuration
8
+
9
+ def self.included(thor)
10
+ thor.class_eval do
11
+ desc "rule [PATH]", "Validate format of a rule file"
12
+ def rule(path)
13
+ # convert str(YAML) to hash or str(path/YAML file) to hash
14
+ rule = load_rule(path)
15
+
16
+ # validate rule schema
17
+ validate_rule rule
18
+
19
+ puts "Valid format. The input is parsed as the following:"
20
+ puts rule.to_yaml
21
+ end
22
+
23
+ desc "config [PATH]", "Validate format of a config file"
24
+ def config(path)
25
+ # convert str(YAML) to hash or str(path/YAML file) to hash
26
+ config = load_config(path)
27
+
28
+ # validate config schema
29
+ validate_config config
30
+
31
+ puts "Valid format. The input is parsed as the following:"
32
+ puts config.to_yaml
33
+ end
34
+ end
35
+ end
36
+ end
37
+ end
38
+ end
@@ -5,13 +5,13 @@ module Mihari
5
5
  module VirusTotal
6
6
  def self.included(thor)
7
7
  thor.class_eval do
8
- desc "virustotal [IP|DOMAIN]", "VirusTotal resolutions lookup by an ip or domain"
8
+ desc "virustotal [IP|DOMAIN]", "VirusTotal resolutions search by an ip or domain"
9
9
  method_option :title, type: :string, desc: "title"
10
10
  method_option :description, type: :string, desc: "description"
11
11
  method_option :tags, type: :array, desc: "tags"
12
12
  def virustotal(indiactor)
13
13
  with_error_handling do
14
- run_analyzer Analyzers::VirusTotal, query: refang(indiactor), options: options
14
+ run_analyzer Analyzers::VirusTotal, query: indiactor, options: options
15
15
  end
16
16
  end
17
17
  end
@@ -5,7 +5,7 @@ module Mihari
5
5
  module ZoomEye
6
6
  def self.included(thor)
7
7
  thor.class_eval do
8
- desc "zoomeye [QUERY]", "ZoomEye search by a query"
8
+ desc "zoomeye [QUERY]", "ZoomEye search"
9
9
  method_option :title, type: :string, desc: "title"
10
10
  method_option :description, type: :string, desc: "description"
11
11
  method_option :tags, type: :array, desc: "tags"
@@ -0,0 +1,5 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Mihari
4
+ ALLOWED_DATA_TYPES = ["hash", "ip", "domain", "url", "mail"].freeze
5
+ end
@@ -2,7 +2,7 @@
2
2
 
3
3
  require "active_record"
4
4
 
5
- class InitialSchema < ActiveRecord::Migration[6.0]
5
+ class InitialSchema < ActiveRecord::Migration[6.1]
6
6
  def change
7
7
  create_table :tags, if_not_exists: true do |t|
8
8
  t.string :name, null: false
@@ -32,6 +32,12 @@ class InitialSchema < ActiveRecord::Migration[6.0]
32
32
  end
33
33
  end
34
34
 
35
+ class V3Schema < ActiveRecord::Migration[6.1]
36
+ def change
37
+ add_column :artifacts, :source, :string, if_not_exists: true
38
+ end
39
+ end
40
+
35
41
  def adapter
36
42
  return "postgresql" if Mihari.config.database.start_with?("postgresql://", "postgres://")
37
43
  return "mysql2" if Mihari.config.database.start_with?("mysql2://")
@@ -54,7 +60,9 @@ module Mihari
54
60
  end
55
61
 
56
62
  ActiveRecord::Migration.verbose = false
63
+
57
64
  InitialSchema.migrate(:up)
65
+ V3Schema.migrate(:up)
58
66
  rescue StandardError
59
67
  # Do nothing
60
68
  end
@@ -65,7 +73,10 @@ module Mihari
65
73
  end
66
74
 
67
75
  def destroy!
68
- InitialSchema.migrate(:down) if ActiveRecord::Base.connected?
76
+ return unless ActiveRecord::Base.connected?
77
+
78
+ InitialSchema.migrate(:down)
79
+ V3Schema.migrate(:down)
69
80
  end
70
81
  end
71
82
  end
@@ -3,8 +3,8 @@
3
3
  module Mihari
4
4
  module Emitters
5
5
  class Base
6
- include Configurable
7
- include Retriable
6
+ include Mixins::Configurable
7
+ include Mixins::Retriable
8
8
 
9
9
  def self.inherited(child)
10
10
  Mihari.emitters << child
@@ -10,7 +10,7 @@ module Mihari
10
10
  def emit(title:, description:, artifacts:, source:, tags: [])
11
11
  return if artifacts.empty?
12
12
 
13
- tags = tags.map { |name| Tag.find_or_create_by(name: name) }.compact.uniq
13
+ tags = tags.filter_map { |name| Tag.find_or_create_by(name: name) }.uniq
14
14
  taggings = tags.map { |tag| Tagging.new(tag_id: tag.id) }
15
15
 
16
16
  alert = Alert.new(
@@ -36,7 +36,7 @@ module Mihari
36
36
 
37
37
  private
38
38
 
39
- def config_keys
39
+ def configuration_keys
40
40
  %w[misp_api_endpoint misp_api_key]
41
41
  end
42
42
 
@@ -2,21 +2,17 @@
2
2
 
3
3
  require "slack-notifier"
4
4
  require "digest/sha2"
5
- require "mem"
6
-
7
- require "mihari/slack_monkeypatch"
5
+ require "dry-initializer"
8
6
 
9
7
  module Mihari
10
8
  module Emitters
11
9
  class Attachment
12
10
  include Mem
13
11
 
14
- attr_reader :data, :data_type
12
+ extend Dry::Initializer
15
13
 
16
- def initialize(data:, data_type:)
17
- @data = data
18
- @data_type = data_type
19
- end
14
+ option :data
15
+ option :data_type
20
16
 
21
17
  def actions
22
18
  [vt_link, urlscan_link, censys_link, shodan_link].compact
@@ -147,7 +143,7 @@ module Mihari
147
143
 
148
144
  private
149
145
 
150
- def config_keys
146
+ def configuration_keys
151
147
  %w[slack_webhook_url]
152
148
  end
153
149
  end
@@ -26,7 +26,7 @@ module Mihari
26
26
 
27
27
  private
28
28
 
29
- def config_keys
29
+ def configuration_keys
30
30
  %w[thehive_api_endpoint thehive_api_key]
31
31
  end
32
32
 
@@ -0,0 +1,53 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+ require "net/http"
5
+ require "uri"
6
+
7
+ module Mihari
8
+ module Emitters
9
+ class Webhook < Base
10
+ # @return [true, false]
11
+ def valid?
12
+ webhook_url?
13
+ end
14
+
15
+ def emit(title:, description:, artifacts:, source:, tags:)
16
+ return if artifacts.empty?
17
+
18
+ uri = URI(Mihari.config.webhook_url)
19
+ data = {
20
+ title: title,
21
+ description: description,
22
+ artifacts: artifacts.map(&:data),
23
+ source: source,
24
+ tags: tags
25
+ }
26
+
27
+ if use_json_body
28
+ Net::HTTP.post(uri, data.to_json, "Content-Type" => "application/json")
29
+ else
30
+ Net::HTTP.post_form(uri, data)
31
+ end
32
+ end
33
+
34
+ private
35
+
36
+ def configuration_keys
37
+ %w[webhook_url]
38
+ end
39
+
40
+ def webhook_url
41
+ @webhook_url ||= Mihari.config.webhook_url
42
+ end
43
+
44
+ def webhook_url?
45
+ !webhook_url.nil?
46
+ end
47
+
48
+ def use_json_body
49
+ @use_json_body ||= Mihari.config.webhook_use_json_body
50
+ end
51
+ end
52
+ end
53
+ end
@@ -0,0 +1,38 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Mihari
4
+ module Mixins
5
+ module Configurable
6
+ #
7
+ # Check whether it is configured or not
8
+ #
9
+ # @return [Boolean]
10
+ #
11
+ def configured?
12
+ configuration_keys.all? { |key| Mihari.config.send(key) }
13
+ end
14
+
15
+ #
16
+ # Configuration values
17
+ #
18
+ # @return [Array<Hash>, nil] Configuration values as a list of hash. Returns nil if there is any keys.
19
+ #
20
+ def configuration_values
21
+ return nil if configuration_keys.empty?
22
+
23
+ configuration_keys.map do |key|
24
+ { key: key.upcase, value: Mihari.config.send(key) }
25
+ end
26
+ end
27
+
28
+ #
29
+ # Configuration keys
30
+ #
31
+ # @return [Array<String>] A list of cofiguration keys
32
+ #
33
+ def configuration_keys
34
+ []
35
+ end
36
+ end
37
+ end
38
+ end
@@ -0,0 +1,85 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "colorize"
4
+ require "yaml"
5
+
6
+ module Mihari
7
+ module Mixins
8
+ module Configuration
9
+ #
10
+ # Load config file into hash
11
+ #
12
+ # @param [String] path Path to YAML file
13
+ #
14
+ # @return [Hash]
15
+ #
16
+ def load_config(path)
17
+ return YAML.safe_load(File.read(path), symbolize_names: true) if Pathname(path).exist?
18
+
19
+ YAML.safe_load(path, symbolize_names: true)
20
+ end
21
+
22
+ #
23
+ # Validate config schema
24
+ #
25
+ # @param [Hash] config
26
+ #
27
+ def validate_config(config)
28
+ error_message = "Failed to parse the input as a config!"
29
+
30
+ contract = Schemas::ConfigurationContract.new
31
+ result = contract.call(config)
32
+ unless result.errors.empty?
33
+ puts error_message.colorize(:red)
34
+ show_validation_errors result.errors
35
+ raise ArgumentError, "Invalid config schema"
36
+ end
37
+
38
+ # check keys
39
+ # TODO: check keys with dry-schema
40
+ valid_keys = Mihari.config.values.keys
41
+ config.each_key do |key|
42
+ unless valid_keys.include?(key)
43
+ puts error_message.colorize(:red)
44
+ raise ArgumentError, "#{key} is not a valid key."
45
+ end
46
+ end
47
+ end
48
+
49
+ #
50
+ # Returns a template for config
51
+ #
52
+ # @return [String] A template for config
53
+ #
54
+ def config_template
55
+ config = Mihari.config.values.keys.map do |key|
56
+ [key.to_s, nil]
57
+ end.to_h
58
+
59
+ YAML.dump(config)
60
+ end
61
+
62
+ #
63
+ # Create (blank) config file
64
+ #
65
+ # @param [String] filename
66
+ # @param [Dry::Files] files
67
+ # @param [String] template
68
+ #
69
+ # @return [nil]
70
+ #
71
+ def initialize_config_yaml(filename, files = Dry::Files.new, template: config_template)
72
+ files.write(filename, template)
73
+ end
74
+
75
+ private
76
+
77
+ def show_validation_errors(errors)
78
+ errors.messages.each do |message|
79
+ path = message.path.map(&:to_s).join
80
+ puts "- #{path} #{message.text}".colorize(:red)
81
+ end
82
+ end
83
+ end
84
+ end
85
+ end
@@ -0,0 +1,20 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "cymbal"
4
+
5
+ module Mihari
6
+ module Mixins
7
+ module Hash
8
+ #
9
+ # Symbolize hash keys
10
+ #
11
+ # @param [Hash] hash
12
+ #
13
+ # @return [Hash]
14
+ #
15
+ def symbolize_hash(hash)
16
+ Cymbal.symbolize hash
17
+ end
18
+ end
19
+ end
20
+ end