mihari 2.4.0 → 3.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (113) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +7 -0
  3. data/.overcommit.yml +12 -0
  4. data/README.md +1 -9
  5. data/build_frontend.sh +5 -0
  6. data/docker/Dockerfile +1 -1
  7. data/exe/mihari +1 -1
  8. data/lib/mihari.rb +89 -15
  9. data/lib/mihari/analyzers/base.rb +49 -8
  10. data/lib/mihari/analyzers/basic.rb +1 -2
  11. data/lib/mihari/analyzers/binaryedge.rb +7 -13
  12. data/lib/mihari/analyzers/censys.rb +26 -63
  13. data/lib/mihari/analyzers/circl.rb +20 -17
  14. data/lib/mihari/analyzers/crtsh.rb +6 -13
  15. data/lib/mihari/analyzers/dnpedia.rb +6 -12
  16. data/lib/mihari/analyzers/dnstwister.rb +13 -10
  17. data/lib/mihari/analyzers/onyphe.rb +6 -12
  18. data/lib/mihari/analyzers/otx.rb +22 -19
  19. data/lib/mihari/analyzers/passivetotal.rb +22 -21
  20. data/lib/mihari/analyzers/pulsedive.rb +16 -13
  21. data/lib/mihari/analyzers/rule.rb +97 -0
  22. data/lib/mihari/analyzers/securitytrails.rb +22 -19
  23. data/lib/mihari/analyzers/shodan.rb +7 -13
  24. data/lib/mihari/analyzers/spyse.rb +12 -19
  25. data/lib/mihari/analyzers/urlscan.rb +22 -27
  26. data/lib/mihari/analyzers/virustotal.rb +25 -22
  27. data/lib/mihari/analyzers/zoomeye.rb +14 -20
  28. data/lib/mihari/cli/analyzer.rb +44 -0
  29. data/lib/mihari/cli/base.rb +27 -0
  30. data/lib/mihari/cli/init.rb +13 -0
  31. data/lib/mihari/cli/main.rb +30 -0
  32. data/lib/mihari/cli/mixins/utils.rb +88 -0
  33. data/lib/mihari/cli/validator.rb +11 -0
  34. data/lib/mihari/commands/binaryedge.rb +1 -1
  35. data/lib/mihari/commands/censys.rb +1 -1
  36. data/lib/mihari/commands/circl.rb +2 -2
  37. data/lib/mihari/commands/crtsh.rb +1 -1
  38. data/lib/mihari/commands/dnpedia.rb +1 -1
  39. data/lib/mihari/commands/dnstwister.rb +2 -2
  40. data/lib/mihari/commands/init.rb +46 -0
  41. data/lib/mihari/commands/json.rb +1 -1
  42. data/lib/mihari/commands/onyphe.rb +1 -1
  43. data/lib/mihari/commands/otx.rb +2 -2
  44. data/lib/mihari/commands/passivetotal.rb +2 -2
  45. data/lib/mihari/commands/pulsedive.rb +2 -2
  46. data/lib/mihari/commands/search.rb +77 -0
  47. data/lib/mihari/commands/securitytrails.rb +2 -2
  48. data/lib/mihari/commands/shodan.rb +1 -1
  49. data/lib/mihari/commands/spyse.rb +1 -1
  50. data/lib/mihari/commands/urlscan.rb +2 -2
  51. data/lib/mihari/commands/validator.rb +38 -0
  52. data/lib/mihari/commands/virustotal.rb +2 -2
  53. data/lib/mihari/commands/zoomeye.rb +1 -1
  54. data/lib/mihari/constraints.rb +5 -0
  55. data/lib/mihari/database.rb +13 -2
  56. data/lib/mihari/emitters/base.rb +2 -2
  57. data/lib/mihari/emitters/database.rb +1 -1
  58. data/lib/mihari/emitters/misp.rb +3 -1
  59. data/lib/mihari/emitters/slack.rb +6 -7
  60. data/lib/mihari/emitters/the_hive.rb +1 -1
  61. data/lib/mihari/emitters/webhook.rb +2 -9
  62. data/lib/mihari/mixins/configurable.rb +38 -0
  63. data/lib/mihari/mixins/configuration.rb +90 -0
  64. data/lib/mihari/mixins/hash.rb +20 -0
  65. data/lib/mihari/mixins/refang.rb +21 -0
  66. data/lib/mihari/mixins/retriable.rb +27 -0
  67. data/lib/mihari/mixins/rule.rb +79 -0
  68. data/lib/mihari/models/alert.rb +28 -1
  69. data/lib/mihari/models/artifact.rb +11 -1
  70. data/lib/mihari/notifiers/base.rb +9 -1
  71. data/lib/mihari/notifiers/exception_notifier.rb +50 -0
  72. data/lib/mihari/notifiers/slack.rb +29 -0
  73. data/lib/mihari/schemas/analyzer.rb +25 -0
  74. data/lib/mihari/schemas/configuration.rb +42 -0
  75. data/lib/mihari/schemas/macros.rb +17 -0
  76. data/lib/mihari/schemas/rule.rb +72 -0
  77. data/lib/mihari/serializers/artifact.rb +1 -1
  78. data/lib/mihari/status.rb +14 -0
  79. data/lib/mihari/templates/rule.yml.erb +19 -0
  80. data/lib/mihari/type_checker.rb +8 -3
  81. data/lib/mihari/version.rb +1 -1
  82. data/lib/mihari/web/app.rb +2 -1
  83. data/lib/mihari/web/controllers/analyzers_controller.rb +38 -0
  84. data/lib/mihari/web/controllers/base_controller.rb +1 -1
  85. data/lib/mihari/web/public/index.html +1 -21
  86. data/lib/mihari/web/public/redoc-static.html +338 -461
  87. data/lib/mihari/web/public/static/js/app.365f1907.js +13 -0
  88. data/lib/mihari/web/public/static/js/app.365f1907.js.map +1 -0
  89. data/lib/mihari/web/public/static/js/app.ab213f7c.js +12 -0
  90. data/lib/mihari/web/public/static/js/app.ab213f7c.js.map +1 -0
  91. data/mihari.gemspec +16 -9
  92. metadata +135 -58
  93. data/.rubocop.yml +0 -161
  94. data/lib/mihari/analyzers/free_text.rb +0 -48
  95. data/lib/mihari/analyzers/http_hash.rb +0 -100
  96. data/lib/mihari/analyzers/passive_dns.rb +0 -59
  97. data/lib/mihari/analyzers/passive_ssl.rb +0 -55
  98. data/lib/mihari/analyzers/reverse_whois.rb +0 -55
  99. data/lib/mihari/analyzers/securitytrails_domain_feed.rb +0 -59
  100. data/lib/mihari/analyzers/ssh_fingerprint.rb +0 -58
  101. data/lib/mihari/cli.rb +0 -126
  102. data/lib/mihari/commands/config.rb +0 -27
  103. data/lib/mihari/commands/free_text.rb +0 -21
  104. data/lib/mihari/commands/http_hash.rb +0 -25
  105. data/lib/mihari/commands/passive_dns.rb +0 -21
  106. data/lib/mihari/commands/passive_ssl.rb +0 -21
  107. data/lib/mihari/commands/reverse_whois.rb +0 -21
  108. data/lib/mihari/commands/securitytrails_domain_feed.rb +0 -23
  109. data/lib/mihari/commands/ssh_fingerprint.rb +0 -21
  110. data/lib/mihari/config.rb +0 -85
  111. data/lib/mihari/configurable.rb +0 -21
  112. data/lib/mihari/html.rb +0 -43
  113. data/lib/mihari/retriable.rb +0 -17
@@ -5,26 +5,29 @@ require "securitytrails"
5
5
  module Mihari
6
6
  module Analyzers
7
7
  class SecurityTrails < Base
8
- attr_reader :query, :type, :title, :description, :tags
8
+ include Mixins::Refang
9
9
 
10
- def initialize(query, title: nil, description: nil, tags: [])
11
- super()
10
+ param :query
11
+ option :title, default: proc { "SecurityTrails search" }
12
+ option :description, default: proc { "query = #{query}" }
13
+ option :tags, default: proc { [] }
12
14
 
13
- @query = query
14
- @type = TypeChecker.type(query)
15
+ attr_reader :type
16
+
17
+ def initialize(*args, **kwargs)
18
+ super
15
19
 
16
- @title = title || "SecurityTrails lookup"
17
- @description = description || "query = #{query}"
18
- @tags = tags
20
+ @query = refang(query)
21
+ @type = TypeChecker.type(query)
19
22
  end
20
23
 
21
24
  def artifacts
22
- lookup || []
25
+ search || []
23
26
  end
24
27
 
25
28
  private
26
29
 
27
- def config_keys
30
+ def configuration_keys
28
31
  %w[securitytrails_api_key]
29
32
  end
30
33
 
@@ -36,20 +39,20 @@ module Mihari
36
39
  %w[ip domain mail].include? type
37
40
  end
38
41
 
39
- def lookup
42
+ def search
40
43
  case type
41
44
  when "domain"
42
- domain_lookup
45
+ domain_search
43
46
  when "ip"
44
- ip_lookup
47
+ ip_search
45
48
  when "mail"
46
- mail_lookup
49
+ mail_search
47
50
  else
48
51
  raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
49
52
  end
50
53
  end
51
54
 
52
- def domain_lookup
55
+ def domain_search
53
56
  result = api.history.get_all_dns_history(query, type: "a")
54
57
  records = result["records"] || []
55
58
  records.map do |record|
@@ -57,16 +60,16 @@ module Mihari
57
60
  end.flatten.compact.uniq
58
61
  end
59
62
 
60
- def ip_lookup
63
+ def ip_search
61
64
  result = api.domains.search(filter: { ipv4: query })
62
65
  records = result["records"] || []
63
- records.map { |record| record["hostname"] }.compact.uniq
66
+ records.filter_map { |record| record["hostname"] }.uniq
64
67
  end
65
68
 
66
- def mail_lookup
69
+ def mail_search
67
70
  result = api.domains.search(filter: { whois_email: query })
68
71
  records = result["records"] || []
69
- records.map { |record| record["hostname"] }.compact.uniq
72
+ records.filter_map { |record| record["hostname"] }.uniq
70
73
  end
71
74
  end
72
75
  end
@@ -5,16 +5,10 @@ require "shodan"
5
5
  module Mihari
6
6
  module Analyzers
7
7
  class Shodan < Base
8
- attr_reader :title, :description, :query, :tags
9
-
10
- def initialize(query, title: nil, description: nil, tags: [])
11
- super()
12
-
13
- @query = query
14
- @title = title || "Shodan lookup"
15
- @description = description || "query = #{query}"
16
- @tags = tags
17
- end
8
+ param :query
9
+ option :title, default: proc { "Shodan search" }
10
+ option :description, default: proc { "query = #{query}" }
11
+ option :tags, default: proc { [] }
18
12
 
19
13
  def artifacts
20
14
  results = search
@@ -22,9 +16,9 @@ module Mihari
22
16
 
23
17
  results.map do |result|
24
18
  matches = result["matches"] || []
25
- matches.map do |match|
19
+ matches.filter_map do |match|
26
20
  match["ip_str"]
27
- end.compact
21
+ end
28
22
  end.flatten.compact.uniq
29
23
  end
30
24
 
@@ -32,7 +26,7 @@ module Mihari
32
26
 
33
27
  PAGE_SIZE = 100
34
28
 
35
- def config_keys
29
+ def configuration_keys
36
30
  %w[shodan_api_key]
37
31
  end
38
32
 
@@ -6,21 +6,14 @@ require "json"
6
6
  module Mihari
7
7
  module Analyzers
8
8
  class Spyse < Base
9
- attr_reader :query, :type, :title, :description, :tags
10
-
11
- def initialize(query, title: nil, description: nil, tags: [], type: "domain")
12
- super()
13
-
14
- @query = query
15
-
16
- @title = title || "Spyse lookup"
17
- @description = description || "query = #{query}"
18
- @tags = tags
19
- @type = type
20
- end
9
+ param :query
10
+ option :title, default: proc { "Spyse search" }
11
+ option :description, default: proc { "query = #{query}" }
12
+ option :type, default: proc { "domain" }
13
+ option :tags, default: proc { [] }
21
14
 
22
15
  def artifacts
23
- lookup || []
16
+ search || []
24
17
  end
25
18
 
26
19
  private
@@ -29,7 +22,7 @@ module Mihari
29
22
  @search_params ||= JSON.parse(query)
30
23
  end
31
24
 
32
- def config_keys
25
+ def configuration_keys
33
26
  %w[spyse_api_key]
34
27
  end
35
28
 
@@ -41,7 +34,7 @@ module Mihari
41
34
  %w[ip domain cert].include? type
42
35
  end
43
36
 
44
- def domain_lookup
37
+ def domain_search
45
38
  res = api.domain.search(search_params, limit: 100)
46
39
  items = res.dig("data", "items") || []
47
40
  items.map do |item|
@@ -49,7 +42,7 @@ module Mihari
49
42
  end.uniq.compact
50
43
  end
51
44
 
52
- def ip_lookup
45
+ def ip_search
53
46
  res = api.ip.search(search_params, limit: 100)
54
47
  items = res.dig("data", "items") || []
55
48
  items.map do |item|
@@ -57,12 +50,12 @@ module Mihari
57
50
  end.uniq.compact
58
51
  end
59
52
 
60
- def lookup
53
+ def search
61
54
  case type
62
55
  when "domain"
63
- domain_lookup
56
+ domain_search
64
57
  when "ip"
65
- ip_lookup
58
+ ip_search
66
59
  else
67
60
  raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
68
61
  end
@@ -2,30 +2,22 @@
2
2
 
3
3
  require "urlscan"
4
4
 
5
+ SUPPORTED_DATA_TYPES = %w[url domain ip].freeze
6
+
5
7
  module Mihari
6
8
  module Analyzers
7
9
  class Urlscan < Base
8
- attr_reader :title, :description, :query, :tags, :target_type, :use_similarity
9
-
10
- def initialize(
11
- query,
12
- description: nil,
13
- tags: [],
14
- target_type: "url",
15
- title: nil,
16
- use_similarity: false
17
- )
18
- super()
19
-
20
- @query = query
21
- @title = title || "urlscan lookup"
22
- @description = description || "query = #{query}"
23
- @tags = tags
24
-
25
- @target_type = target_type
26
- @use_similarity = use_similarity
27
-
28
- raise InvalidInputError, "type should be url, domain or ip." unless valid_target_type?
10
+ param :query
11
+ option :title, default: proc { "urlscan search" }
12
+ option :description, default: proc { "query = #{query}" }
13
+ option :tags, default: proc { [] }
14
+ option :allowed_data_types, default: proc { SUPPORTED_DATA_TYPES }
15
+ option :use_similarity, default: proc { false }
16
+
17
+ def initialize(*args, **kwargs)
18
+ super
19
+
20
+ raise InvalidInputError, "allowed_data_types should be any of url, domain and ip." unless valid_alllowed_data_types?
29
21
  end
30
22
 
31
23
  def artifacts
@@ -33,14 +25,17 @@ module Mihari
33
25
  return [] unless result
34
26
 
35
27
  results = result["results"] || []
36
- results.map do |match|
37
- match.dig "page", target_type
38
- end.compact.uniq
28
+
29
+ allowed_data_types.map do |type|
30
+ results.filter_map do |match|
31
+ match.dig "page", type
32
+ end.uniq
33
+ end.flatten
39
34
  end
40
35
 
41
36
  private
42
37
 
43
- def config_keys
38
+ def configuration_keys
44
39
  %w[urlscan_api_key]
45
40
  end
46
41
 
@@ -54,8 +49,8 @@ module Mihari
54
49
  api.search(query, size: 10_000)
55
50
  end
56
51
 
57
- def valid_target_type?
58
- %w[url domain ip].include? target_type
52
+ def valid_alllowed_data_types?
53
+ allowed_data_types.all? { |type| SUPPORTED_DATA_TYPES.include? type }
59
54
  end
60
55
  end
61
56
  end
@@ -5,26 +5,29 @@ require "virustotal"
5
5
  module Mihari
6
6
  module Analyzers
7
7
  class VirusTotal < Base
8
- attr_reader :indicator, :type, :title, :description, :tags
8
+ include Mixins::Refang
9
9
 
10
- def initialize(indicator, title: nil, description: nil, tags: [])
11
- super()
10
+ param :query
11
+ option :title, default: proc { "VirusTotal search" }
12
+ option :description, default: proc { "query = #{query}" }
13
+ option :tags, default: proc { [] }
12
14
 
13
- @indicator = indicator
14
- @type = TypeChecker.type(indicator)
15
+ attr_reader :type
15
16
 
16
- @title = title || "VirusTotal lookup"
17
- @description = description || "indicator = #{indicator}"
18
- @tags = tags
17
+ def initialize(*args, **kwargs)
18
+ super
19
+
20
+ @query = refang(query)
21
+ @type = TypeChecker.type(query)
19
22
  end
20
23
 
21
24
  def artifacts
22
- lookup || []
25
+ search || []
23
26
  end
24
27
 
25
28
  private
26
29
 
27
- def config_keys
30
+ def configuration_keys
28
31
  %w[virustotal_api_key]
29
32
  end
30
33
 
@@ -36,33 +39,33 @@ module Mihari
36
39
  %w[ip domain].include? type
37
40
  end
38
41
 
39
- def lookup
42
+ def search
40
43
  case type
41
44
  when "domain"
42
- domain_lookup
45
+ domain_search
43
46
  when "ip"
44
- ip_lookup
47
+ ip_search
45
48
  else
46
- raise InvalidInputError, "#{indicator}(type: #{type || "unknown"}) is not supported." unless valid_type?
49
+ raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
47
50
  end
48
51
  end
49
52
 
50
- def domain_lookup
51
- res = api.domain.resolutions(indicator)
53
+ def domain_search
54
+ res = api.domain.resolutions(query)
52
55
 
53
56
  data = res["data"] || []
54
- data.map do |item|
57
+ data.filter_map do |item|
55
58
  item.dig("attributes", "ip_address")
56
- end.compact.uniq
59
+ end.uniq
57
60
  end
58
61
 
59
- def ip_lookup
60
- res = api.ip_address.resolutions(indicator)
62
+ def ip_search
63
+ res = api.ip_address.resolutions(query)
61
64
 
62
65
  data = res["data"] || []
63
- data.map do |item|
66
+ data.filter_map do |item|
64
67
  item.dig("attributes", "host_name")
65
- end.compact.uniq
68
+ end.uniq
66
69
  end
67
70
  end
68
71
  end
@@ -5,24 +5,18 @@ require "zoomeye"
5
5
  module Mihari
6
6
  module Analyzers
7
7
  class ZoomEye < Base
8
- attr_reader :title, :description, :query, :tags, :type
9
-
10
- def initialize(query, title: nil, description: nil, tags: [], type: "host")
11
- super()
12
-
13
- @query = query
14
- @title = title || "ZoomEye lookup"
15
- @description = description || "query = #{query}"
16
- @tags = tags
17
- @type = type
18
- end
8
+ param :query
9
+ option :title, default: proc { "ZoomEye search" }
10
+ option :description, default: proc { "query = #{query}" }
11
+ option :tags, default: proc { [] }
12
+ option :type, default: proc { "host" }
19
13
 
20
14
  def artifacts
21
15
  case type
22
16
  when "host"
23
- host_lookup
17
+ host_search
24
18
  when "web"
25
- web_lookup
19
+ web_search
26
20
  else
27
21
  raise InvalidInputError, "#{type} type is not supported." unless valid_type?
28
22
  end
@@ -36,7 +30,7 @@ module Mihari
36
30
  %w[host web].include? type
37
31
  end
38
32
 
39
- def config_keys
33
+ def configuration_keys
40
34
  %w[zoomeye_api_key]
41
35
  end
42
36
 
@@ -53,16 +47,16 @@ module Mihari
53
47
  end.flatten.compact.uniq
54
48
  end
55
49
 
56
- def _host_lookup(query, page: 1)
50
+ def _host_search(query, page: 1)
57
51
  api.host.search(query, page: page)
58
52
  rescue ::ZoomEye::Error => _e
59
53
  nil
60
54
  end
61
55
 
62
- def host_lookup
56
+ def host_search
63
57
  responses = []
64
58
  (1..Float::INFINITY).each do |page|
65
- res = _host_lookup(query, page: page)
59
+ res = _host_search(query, page: page)
66
60
  break unless res
67
61
 
68
62
  total = res["total"].to_i
@@ -72,16 +66,16 @@ module Mihari
72
66
  convert_responses responses.compact
73
67
  end
74
68
 
75
- def _web_lookup(query, page: 1)
69
+ def _web_search(query, page: 1)
76
70
  api.web.search(query, page: page)
77
71
  rescue ::ZoomEye::Error => _e
78
72
  nil
79
73
  end
80
74
 
81
- def web_lookup
75
+ def web_search
82
76
  responses = []
83
77
  (1..Float::INFINITY).each do |page|
84
- res = _web_lookup(query, page: page)
78
+ res = _web_search(query, page: page)
85
79
  break unless res
86
80
 
87
81
  total = res["total"].to_i
@@ -0,0 +1,44 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "mihari/commands/binaryedge"
4
+ require "mihari/commands/censys"
5
+ require "mihari/commands/circl"
6
+ require "mihari/commands/crtsh"
7
+ require "mihari/commands/dnpedia"
8
+ require "mihari/commands/dnstwister"
9
+ require "mihari/commands/onyphe"
10
+ require "mihari/commands/otx"
11
+ require "mihari/commands/passivetotal"
12
+ require "mihari/commands/pulsedive"
13
+ require "mihari/commands/securitytrails"
14
+ require "mihari/commands/shodan"
15
+ require "mihari/commands/spyse"
16
+ require "mihari/commands/urlscan"
17
+ require "mihari/commands/virustotal"
18
+ require "mihari/commands/zoomeye"
19
+
20
+ require "mihari/commands/json"
21
+
22
+ module Mihari
23
+ module CLI
24
+ class Analyzer < Base
25
+ include Mihari::Commands::BinaryEdge
26
+ include Mihari::Commands::Censys
27
+ include Mihari::Commands::CIRCL
28
+ include Mihari::Commands::Crtsh
29
+ include Mihari::Commands::DNPedia
30
+ include Mihari::Commands::DNSTwister
31
+ include Mihari::Commands::JSON
32
+ include Mihari::Commands::Onyphe
33
+ include Mihari::Commands::OTX
34
+ include Mihari::Commands::PassiveTotal
35
+ include Mihari::Commands::Pulsedive
36
+ include Mihari::Commands::SecurityTrails
37
+ include Mihari::Commands::Shodan
38
+ include Mihari::Commands::Spyse
39
+ include Mihari::Commands::Urlscan
40
+ include Mihari::Commands::VirusTotal
41
+ include Mihari::Commands::ZoomEye
42
+ end
43
+ end
44
+ end