mihari 6.3.0 → 7.0.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (161) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +4 -10
  3. data/.rubocop.yml +2 -0
  4. data/Dockerfile +14 -0
  5. data/config.ru +5 -3
  6. data/docker-compose.yml +61 -0
  7. data/exe/mihari +2 -1
  8. data/lefthook.yml +8 -0
  9. data/lib/mihari/actor.rb +4 -4
  10. data/lib/mihari/analyzers/base.rb +16 -0
  11. data/lib/mihari/analyzers/binaryedge.rb +4 -2
  12. data/lib/mihari/analyzers/censys.rb +7 -5
  13. data/lib/mihari/analyzers/circl.rb +5 -3
  14. data/lib/mihari/analyzers/crtsh.rb +10 -2
  15. data/lib/mihari/analyzers/dnstwister.rb +1 -1
  16. data/lib/mihari/analyzers/feed.rb +12 -20
  17. data/lib/mihari/analyzers/fofa.rb +6 -8
  18. data/lib/mihari/analyzers/greynoise.rb +4 -2
  19. data/lib/mihari/analyzers/hunterhow.rb +4 -2
  20. data/lib/mihari/analyzers/onyphe.rb +4 -2
  21. data/lib/mihari/analyzers/otx.rb +5 -3
  22. data/lib/mihari/analyzers/passivetotal.rb +29 -12
  23. data/lib/mihari/analyzers/pulsedive.rb +5 -3
  24. data/lib/mihari/analyzers/securitytrails.rb +32 -8
  25. data/lib/mihari/analyzers/shodan.rb +4 -2
  26. data/lib/mihari/analyzers/urlscan.rb +4 -2
  27. data/lib/mihari/analyzers/virustotal.rb +5 -5
  28. data/lib/mihari/analyzers/virustotal_intelligence.rb +4 -2
  29. data/lib/mihari/analyzers/zoomeye.rb +4 -2
  30. data/lib/mihari/cli/{main.rb → application.rb} +17 -5
  31. data/lib/mihari/cli/artifact.rb +14 -0
  32. data/lib/mihari/cli/config.rb +14 -0
  33. data/lib/mihari/cli/rule.rb +1 -0
  34. data/lib/mihari/cli/tag.rb +14 -0
  35. data/lib/mihari/clients/base.rb +2 -2
  36. data/lib/mihari/clients/binaryedge.rb +2 -2
  37. data/lib/mihari/clients/crtsh.rb +3 -10
  38. data/lib/mihari/clients/fofa.rb +1 -1
  39. data/lib/mihari/clients/hunterhow.rb +1 -1
  40. data/lib/mihari/clients/mmdb.rb +28 -0
  41. data/lib/mihari/clients/passivetotal.rb +7 -20
  42. data/lib/mihari/clients/securitytrails.rb +19 -43
  43. data/lib/mihari/clients/shodan_internet_db.rb +28 -0
  44. data/lib/mihari/clients/the_hive.rb +7 -5
  45. data/lib/mihari/commands/alert.rb +53 -11
  46. data/lib/mihari/commands/artifact.rb +66 -0
  47. data/lib/mihari/commands/config.rb +23 -0
  48. data/lib/mihari/commands/database.rb +1 -1
  49. data/lib/mihari/commands/rule.rb +40 -27
  50. data/lib/mihari/commands/search.rb +10 -11
  51. data/lib/mihari/commands/sidekiq.rb +31 -0
  52. data/lib/mihari/commands/tag.rb +46 -0
  53. data/lib/mihari/commands/web.rb +6 -7
  54. data/lib/mihari/{mixins/autonomous_system.rb → concerns/autonomous_system_normalizable.rb} +5 -3
  55. data/lib/mihari/concerns/configurable.rb +72 -0
  56. data/lib/mihari/concerns/database_connectable.rb +16 -0
  57. data/lib/mihari/{mixins/unwrap_error.rb → concerns/error_unwrappable.rb} +5 -3
  58. data/lib/mihari/{mixins/falsepositive.rb → concerns/falsepositive_validatable.rb} +5 -3
  59. data/lib/mihari/{mixins/refang.rb → concerns/refangable.rb} +5 -3
  60. data/lib/mihari/{mixins → concerns}/retriable.rb +4 -2
  61. data/lib/mihari/config.rb +13 -12
  62. data/lib/mihari/database.rb +30 -42
  63. data/lib/mihari/emitters/database.rb +5 -6
  64. data/lib/mihari/emitters/misp.rb +4 -11
  65. data/lib/mihari/emitters/slack.rb +7 -5
  66. data/lib/mihari/emitters/the_hive.rb +8 -58
  67. data/lib/mihari/emitters/webhook.rb +6 -6
  68. data/lib/mihari/enrichers/google_public_dns.rb +1 -1
  69. data/lib/mihari/enrichers/mmdb.rb +28 -0
  70. data/lib/mihari/enrichers/shodan.rb +3 -5
  71. data/lib/mihari/enrichers/whois.rb +3 -3
  72. data/lib/mihari/entities/alert.rb +3 -10
  73. data/lib/mihari/entities/artifact.rb +6 -14
  74. data/lib/mihari/entities/config.rb +2 -2
  75. data/lib/mihari/entities/cpe.rb +1 -0
  76. data/lib/mihari/entities/dns.rb +1 -0
  77. data/lib/mihari/entities/geolocation.rb +1 -0
  78. data/lib/mihari/entities/ip_address.rb +1 -3
  79. data/lib/mihari/entities/messages.rb +17 -0
  80. data/lib/mihari/entities/pagination.rb +11 -0
  81. data/lib/mihari/entities/port.rb +1 -0
  82. data/lib/mihari/entities/reverse_dns.rb +1 -0
  83. data/lib/mihari/entities/rule.rb +2 -20
  84. data/lib/mihari/entities/tag.rb +2 -2
  85. data/lib/mihari/entities/whois.rb +1 -0
  86. data/lib/mihari/errors.rb +2 -4
  87. data/lib/mihari/http.rb +4 -0
  88. data/lib/mihari/models/alert.rb +21 -53
  89. data/lib/mihari/models/artifact.rb +61 -97
  90. data/lib/mihari/models/autonomous_system.rb +0 -24
  91. data/lib/mihari/models/concerns/searchable.rb +50 -0
  92. data/lib/mihari/models/cpe.rb +0 -23
  93. data/lib/mihari/models/dns.rb +0 -20
  94. data/lib/mihari/models/geolocation.rb +0 -24
  95. data/lib/mihari/models/port.rb +3 -10
  96. data/lib/mihari/models/reverse_dns.rb +0 -23
  97. data/lib/mihari/models/rule.rb +16 -57
  98. data/lib/mihari/models/tag.rb +17 -1
  99. data/lib/mihari/models/tagging.rb +1 -1
  100. data/lib/mihari/models/whois.rb +0 -17
  101. data/lib/mihari/rule.rb +35 -24
  102. data/lib/mihari/schemas/alert.rb +1 -0
  103. data/lib/mihari/schemas/analyzer.rb +3 -2
  104. data/lib/mihari/schemas/concerns/orrable.rb +24 -0
  105. data/lib/mihari/schemas/emitter.rb +1 -2
  106. data/lib/mihari/schemas/enricher.rb +3 -4
  107. data/lib/mihari/schemas/macros.rb +1 -1
  108. data/lib/mihari/schemas/options.rb +0 -2
  109. data/lib/mihari/schemas/rule.rb +1 -2
  110. data/lib/mihari/services/builders.rb +158 -0
  111. data/lib/mihari/services/creators.rb +22 -0
  112. data/lib/mihari/services/destroyers.rb +41 -0
  113. data/lib/mihari/services/enrichers.rb +25 -0
  114. data/lib/mihari/services/feed.rb +107 -0
  115. data/lib/mihari/services/getters.rb +58 -0
  116. data/lib/mihari/services/initializers.rb +22 -0
  117. data/lib/mihari/services/{alert_builder.rb → proxies.rb} +10 -40
  118. data/lib/mihari/services/searchers.rb +91 -0
  119. data/lib/mihari/sidekiq/application.rb +13 -0
  120. data/lib/mihari/sidekiq/jobs.rb +36 -0
  121. data/lib/mihari/structs/censys.rb +1 -1
  122. data/lib/mihari/structs/config.rb +10 -10
  123. data/lib/mihari/structs/filters.rb +12 -130
  124. data/lib/mihari/structs/google_public_dns.rb +1 -1
  125. data/lib/mihari/structs/greynoise.rb +1 -1
  126. data/lib/mihari/structs/mmdb.rb +115 -0
  127. data/lib/mihari/structs/onyphe.rb +1 -1
  128. data/lib/mihari/structs/shodan.rb +2 -2
  129. data/lib/mihari/version.rb +1 -1
  130. data/lib/mihari/web/{app.rb → application.rb} +28 -15
  131. data/lib/mihari/web/endpoints/alerts.rb +34 -73
  132. data/lib/mihari/web/endpoints/artifacts.rb +27 -111
  133. data/lib/mihari/web/endpoints/configs.rb +3 -5
  134. data/lib/mihari/web/endpoints/ip_addresses.rb +14 -15
  135. data/lib/mihari/web/endpoints/rules.rb +58 -130
  136. data/lib/mihari/web/endpoints/tags.rb +21 -17
  137. data/lib/mihari/web/middleware/capture_exceptions.rb +25 -0
  138. data/lib/mihari/web/middleware/{connection_adapter.rb → connection.rb} +4 -2
  139. data/lib/mihari/web/public/assets/index-cQUcyII5.js +1766 -0
  140. data/lib/mihari/web/public/assets/index-dVaNxqTC.css +1 -0
  141. data/lib/mihari/web/public/index.html +2 -2
  142. data/lib/mihari/web/public/redoc-static.html +385 -385
  143. data/lib/mihari.rb +56 -28
  144. data/mihari.gemspec +12 -4
  145. data/mkdocs.yml +5 -2
  146. data/requirements.txt +1 -1
  147. metadata +164 -34
  148. data/lib/mihari/commands/mixins.rb +0 -11
  149. data/lib/mihari/enrichers/ipinfo.rb +0 -52
  150. data/lib/mihari/entities/message.rb +0 -9
  151. data/lib/mihari/feed/parser.rb +0 -38
  152. data/lib/mihari/feed/reader.rb +0 -111
  153. data/lib/mihari/mixins/configurable.rb +0 -68
  154. data/lib/mihari/schemas/mixins.rb +0 -20
  155. data/lib/mihari/services/alert_runner.rb +0 -20
  156. data/lib/mihari/services/rule_builder.rb +0 -46
  157. data/lib/mihari/structs/ipinfo.rb +0 -53
  158. data/lib/mihari/web/endpoints/exports.rb +0 -0
  159. data/lib/mihari/web/middleware/error_notification_adapter.rb +0 -35
  160. data/lib/mihari/web/public/assets/index-81613_nX.js +0 -1763
  161. data/lib/mihari/web/public/assets/index-Wv6xUrTI.css +0 -1
@@ -1,52 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Mihari
4
- module Enrichers
5
- #
6
- # IPInfo enricher
7
- #
8
- class IPInfo < Base
9
- # @return [String, nil]
10
- attr_reader :api_key
11
-
12
- #
13
- # @param [Hash, nil] options
14
- # @param [String, nil] api_key
15
- #
16
- def initialize(options: nil, api_key: nil)
17
- @api_key = api_key || Mihari.config.ipinfo_api_key
18
-
19
- super(options: options)
20
- end
21
-
22
- def configuration_keys
23
- %w[ipinfo_api_key]
24
- end
25
-
26
- #
27
- # Query IPInfo
28
- #
29
- # @param [String] ip
30
- #
31
- # @return [Mihari::Structs::IPInfo::Response]
32
- #
33
- def call(ip)
34
- url = "https://ipinfo.io/#{ip}/json"
35
- res = http.get(url)
36
- Structs::IPInfo::Response.from_dynamic! JSON.parse(res.body.to_s)
37
- end
38
- memo_wise :call
39
-
40
- private
41
-
42
- def headers
43
- authorization = api_key.nil? ? nil : "Bearer #{api_key}"
44
- { authorization: authorization }.compact
45
- end
46
-
47
- def http
48
- HTTP::Factory.build headers: headers, timeout: timeout
49
- end
50
- end
51
- end
52
- end
@@ -1,9 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Mihari
4
- module Entities
5
- class Message < Grape::Entity
6
- expose :message, documentation: { type: String, required: true }
7
- end
8
- end
9
- end
@@ -1,38 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require "jr/cli/core_ext"
4
-
5
- module Mihari
6
- module Feed
7
- #
8
- # Feed parser
9
- #
10
- class Parser
11
- # @return [Array<Hash>, Array<Array<String>>]
12
- attr_reader :data
13
-
14
- #
15
- # @param [Array<Hash>, Array<Array<String>>] data
16
- #
17
- def initialize(data)
18
- @data = data
19
- end
20
-
21
- #
22
- # Parse data by selector
23
- #
24
- # @param [String] selector
25
- #
26
- # @return [Array<String>]
27
- #
28
- def parse(selector)
29
- parsed = data.instance_eval(selector)
30
-
31
- raise TypeError unless parsed.is_a?(Array) || parsed.is_a?(Enumerator)
32
- raise TypeError unless parsed.all?(String)
33
-
34
- parsed.to_a
35
- end
36
- end
37
- end
38
- end
@@ -1,111 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require "csv"
4
-
5
- module Mihari
6
- module Feed
7
- #
8
- # Feed reader
9
- #
10
- class Reader
11
- # @return [String]
12
- attr_reader :url
13
-
14
- # @return [Hash]
15
- attr_reader :headers
16
-
17
- # @return [Hash, nil]
18
- attr_reader :params
19
-
20
- # @return [Hash, nil]
21
- attr_reader :json
22
-
23
- # @return [Hash, nil]
24
- attr_reader :data
25
-
26
- # @return [String]
27
- attr_reader :method
28
-
29
- # @return [Integer, nil]
30
- attr_reader :timeout
31
-
32
- def initialize(url, headers: {}, method: "GET", params: nil, json: nil, data: nil, timeout: nil)
33
- @url = Addressable::URI.parse(url)
34
- @headers = headers
35
- @method = method
36
- @timeout = timeout
37
-
38
- @params = params
39
- @json = json
40
- @data = data
41
- end
42
-
43
- #
44
- # @return [Array<Hash>]
45
- #
46
- def read
47
- return read_file(url.path) if url.scheme == "file"
48
-
49
- res = nil
50
- res = http.get(url, params: params) if method == "GET"
51
- res = http.post(url, params: params, json: json, form: data) if method == "POST"
52
-
53
- return [] if res.nil?
54
-
55
- body = res.body.to_s
56
- content_type = res["Content-Type"].to_s
57
- return convert_as_json(body) if content_type.include?("application/json")
58
-
59
- convert_as_csv(body)
60
- end
61
-
62
- #
63
- # Convert text as JSON
64
- #
65
- # @param [String] text
66
- #
67
- # @return [Array<Hash>]
68
- #
69
- def convert_as_json(text)
70
- parsed = JSON.parse(text, symbolize_names: true)
71
- return parsed if parsed.is_a?(Array)
72
-
73
- [parsed]
74
- end
75
-
76
- #
77
- # Convert text as CSV
78
- #
79
- # @param [String] text
80
- #
81
- # @return [Array<Hash>]
82
- #
83
- def convert_as_csv(text)
84
- text_without_comments = text.lines.reject { |line| line.start_with? "#" }.join("\n")
85
-
86
- CSV.new(text_without_comments).to_a.reject(&:empty?)
87
- end
88
-
89
- #
90
- # Read & convert a file
91
- #
92
- # @param [String] path
93
- #
94
- # @return [Array<Hash>]
95
- #
96
- def read_file(path)
97
- text = File.read(path)
98
-
99
- return convert_as_json(text) if path.end_with?(".json")
100
-
101
- convert_as_csv text
102
- end
103
-
104
- private
105
-
106
- def http
107
- HTTP::Factory.build headers: headers, timeout: timeout
108
- end
109
- end
110
- end
111
- end
@@ -1,68 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Mihari
4
- module Mixins
5
- #
6
- # Configurable mixin
7
- #
8
- module Configurable
9
- #
10
- # Check whether there are configuration key-values or not
11
- #
12
- # @return [Boolean]
13
- #
14
- def configuration_keys?
15
- return true if configuration_keys.empty?
16
-
17
- configuration_keys.all? { |key| Mihari.config.send(key) }
18
- end
19
-
20
- #
21
- # Check whether it is configured or not
22
- #
23
- # @return [Boolean]
24
- #
25
- def configured?
26
- configuration_keys? || api_key?
27
- end
28
-
29
- #
30
- # Configuration values
31
- #
32
- # @return [Array<Hash>, nil] Configuration values as a list of hash. Returns nil if there is any keys.
33
- #
34
- def configuration_values
35
- return nil if configuration_keys.empty?
36
-
37
- configuration_keys.map do |key|
38
- value = Mihari.config.send(key)
39
- value = "REDACTED" if value && Mihari.config.hide_config_values
40
- { key: key.upcase, value: value }
41
- end
42
- end
43
-
44
- #
45
- # Configuration keys
46
- #
47
- # @return [Array<String>] A list of configuration keys
48
- #
49
- def configuration_keys
50
- []
51
- end
52
-
53
- private
54
-
55
- #
56
- # Check whether API key is set or not
57
- #
58
- # @return [Boolean]
59
- #
60
- def api_key?
61
- value = method(:api_key).call
62
- !value.nil?
63
- rescue NameError
64
- true
65
- end
66
- end
67
- end
68
- end
@@ -1,20 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Mihari
4
- module Schemas
5
- #
6
- # Mixins for schemas
7
- #
8
- module Mixins
9
- def get_or_composition
10
- schemas = constants.map { |sym| const_get sym }
11
- return schemas.first if schemas.length <= 1
12
-
13
- base, *others = schemas
14
- others.each { |other| base = base.or(other) }
15
-
16
- base
17
- end
18
- end
19
- end
20
- end
@@ -1,20 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Mihari
4
- module Services
5
- #
6
- # Alert runner
7
- #
8
- class AlertRunner < Service
9
- #
10
- # @param [Mihari::Services::AlertProxy] alert
11
- #
12
- # @return [Mihari::Models::Alert]
13
- #
14
- def call(alert)
15
- emitter = Emitters::Database.new(rule: alert.rule)
16
- emitter.call alert.artifacts
17
- end
18
- end
19
- end
20
- end
@@ -1,46 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require "date"
4
- require "erb"
5
- require "pathname"
6
- require "yaml"
7
-
8
- module Mihari
9
- module Services
10
- #
11
- # Rule builder
12
- #
13
- class RuleBuilder < Service
14
- # @return [String]
15
- attr_reader :path_or_id
16
-
17
- #
18
- # @param [String] path_or_id
19
- #
20
- # @return [Hash]
21
- #
22
- def data
23
- result = Try { Mihari::Models::Rule.find path_or_id }.to_result
24
- return result.value! if result.success?
25
-
26
- raise ArgumentError, "#{path_or_id} does not exist" unless Pathname(path_or_id).exist?
27
-
28
- YAML.safe_load(
29
- ERB.new(File.read(path_or_id)).result,
30
- permitted_classes: [Date, Symbol]
31
- )
32
- end
33
-
34
- #
35
- # @param [String] path_or_id
36
- #
37
- # @return [Mihari::Rule]
38
- #
39
- def call(path_or_id)
40
- @path_or_id = path_or_id
41
-
42
- Rule.new(**data)
43
- end
44
- end
45
- end
46
- end
@@ -1,53 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Mihari
4
- module Structs
5
- module IPInfo
6
- class Response < Dry::Struct
7
- # @!attribute [r] ip
8
- # @return [String]
9
- attribute :ip, Types::String
10
-
11
- # @!attribute [r] hostname
12
- # @return [String, nil]
13
- attribute :hostname, Types::String.optional
14
-
15
- # @!attribute [r] loc
16
- # @return [String, nil]
17
- attribute :loc, Types::String.optional
18
-
19
- # @!attribute [r] country_code
20
- # @return [String, nil]
21
- attribute :country_code, Types::String.optional
22
-
23
- # @!attribute [r] asn
24
- # @return [Integer, nil]
25
- attribute :asn, Types::Int.optional
26
-
27
- class << self
28
- include Mixins::AutonomousSystem
29
-
30
- #
31
- # @param [Hash] d
32
- #
33
- def from_dynamic!(d)
34
- d = d.deep_stringify_keys
35
- d = Types::Hash[d]
36
-
37
- asn = nil
38
- asn_ = d.dig("asn", "asn")
39
- asn = normalize_asn(asn_) unless asn_.nil?
40
-
41
- new(
42
- ip: d.fetch("ip"),
43
- loc: d["loc"],
44
- hostname: d["hostname"],
45
- country_code: d["country"],
46
- asn: asn
47
- )
48
- end
49
- end
50
- end
51
- end
52
- end
53
- end
File without changes
@@ -1,35 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Mihari
4
- module Web
5
- module Middleware
6
- #
7
- # Error notification adapter for Rack app
8
- #
9
- class ErrorNotificationAdapter
10
- include Mihari::Mixins::UnwrapError
11
-
12
- attr_reader :app
13
-
14
- def initialize(app)
15
- @app = app
16
- end
17
-
18
- def with_error_notification
19
- yield
20
- rescue StandardError => e
21
- unwrapped = unwrap_error(e)
22
-
23
- Mihari.logger.error unwrapped
24
- Sentry.capture_exception(unwrapped) if Sentry.initialized?
25
-
26
- raise unwrapped
27
- end
28
-
29
- def call(env)
30
- with_error_notification { app.call(env) }
31
- end
32
- end
33
- end
34
- end
35
- end