mihari 7.2.0 → 7.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 6b91d99562526b653b793f71e8ef5575f113d26859ca86b91f1980d1c44e898c
4
- data.tar.gz: 07c302ce446b0f0986bfc82dd575ebde203f4b25b73a1aee72a2ce37a08b9b87
3
+ metadata.gz: ed9b24457d01edc4d1643e6c31c654ad7c13bf71fc849b10bb02276abf45852c
4
+ data.tar.gz: 1be36d2083e0b0209ad16475a8f20eb4d2ee9bfcb37eae43fa76091207526def
5
5
  SHA512:
6
- metadata.gz: 67d607a09ab2992b6721358b9e96e0c049a355221b2e97358440d70f96ef4933ac86b337bcc12bdc4b2aafccf4e5d6cf8cd68f4b2bbe567523bfba22b7fbd88c
7
- data.tar.gz: 6f73de19824d31e21bae4ee7ce456c1b15fa0a875d1f07025c33ac0356ef257ac2eb819ffd8b685bf8914a472da04f691e5ae02a361397da9a83253d5345b108
6
+ metadata.gz: 4482938e33386e24054cb215f78f065e3190ba32269872aea6a9f543745a2c71777bb609120dcaa3872dba9b6ebd307651f80239342d6ac9427db205f03c80e0
7
+ data.tar.gz: 5723cbca9f18c519fc4cf91775d751f417a85161add0c0c6d499fd03075c7985c1cb73ef68e24b4913b1d8d3a9652a0e4e1025a32b83c987d3e9e22886923e38
data/Dockerfile CHANGED
@@ -1,4 +1,4 @@
1
- FROM ruby:3.2.2-alpine3.19
1
+ FROM ruby:3.3.0-alpine3.19
2
2
 
3
3
  ARG MIHARI_VERSION=0.0.0
4
4
 
@@ -19,6 +19,13 @@ module Mihari
19
19
  @rule = rule
20
20
  end
21
21
 
22
+ #
23
+ # @return [Boolean]
24
+ #
25
+ def parallel?
26
+ options[:parallel] || Mihari.config.parallel
27
+ end
28
+
22
29
  # A target to emit the data
23
30
  #
24
31
  # @return [String]
@@ -6,46 +6,88 @@ module Mihari
6
6
  # Base class for enrichers
7
7
  #
8
8
  class Base < Actor
9
- prepend MemoWise
10
-
9
+ #
10
+ # @param [Hash, nil] options
11
+ #
11
12
  def initialize(options: nil)
12
13
  super(options: options)
13
14
  end
14
15
 
15
16
  #
16
- # @param [String] value
17
+ # Enrich an artifact
18
+ #
19
+ # @param [Mihari::Models::Artifact] artifact
17
20
  #
18
- def call(value)
21
+ # @return [Mihari::Models::Artifact]
22
+ #
23
+ def call(artifact)
19
24
  raise NotImplementedError, "You must implement #{self.class}##{__method__}"
20
25
  end
21
26
 
22
27
  #
23
- # @param [Mihari::Models::Artifact] value
28
+ # @param [Mihari::Models::Artifact] artifact
24
29
  #
25
30
  # @return [Dry::Monads::Result::Success<Object>, Dry::Monads::Result::Failure]
26
31
  #
27
- def result(value)
32
+ def result(artifact)
33
+ return unless callable?(artifact)
34
+
28
35
  result = Try[StandardError] do
29
- retry_on_error(
30
- times: retry_times,
31
- interval: retry_interval,
32
- exponential_backoff: retry_exponential_backoff
33
- ) { call value }
36
+ retry_on_error(times: retry_times, interval: retry_interval,
37
+ exponential_backoff: retry_exponential_backoff) do
38
+ call artifact
39
+ end
34
40
  end.to_result
35
41
 
36
42
  if result.failure?
37
- Mihari.logger.warn("Enricher:#{self.class.key} for #{value.truncate(32)} failed: #{result.failure}")
43
+ Mihari.logger.warn("Enricher:#{self.class.key} for #{artifact.data.truncate(32)} failed: #{result.failure}")
38
44
  end
39
45
 
40
46
  result
41
47
  end
42
48
 
49
+ #
50
+ # @param [Mihari::Models::Artifact] artifact
51
+ #
52
+ # @return [Boolean]
53
+ #
54
+ def callable?(artifact)
55
+ callable_data_type?(artifact) && callable_relationships?(artifact)
56
+ end
57
+
43
58
  class << self
44
59
  def inherited(child)
45
60
  super
46
61
  Mihari.enrichers << child
47
62
  end
48
63
  end
64
+
65
+ private
66
+
67
+ #
68
+ # @param [Mihari::Models::Artifact] artifact
69
+ #
70
+ # @return [Boolean]
71
+ #
72
+ def callable_data_type?(artifact)
73
+ supported_data_types.include? artifact.data_type
74
+ end
75
+
76
+ #
77
+ # @param [Mihari::Models::Artifact] artifact
78
+ #
79
+ # @return [Boolean]
80
+ #
81
+ def callable_relationships?(artifact)
82
+ raise NotImplementedError, "You must implement #{self.class}##{__method__}"
83
+ end
84
+
85
+ #
86
+ # @return [Array<String>]
87
+ #
88
+ def supported_data_types
89
+ []
90
+ end
49
91
  end
50
92
  end
51
93
  end
@@ -7,14 +7,22 @@ module Mihari
7
7
  #
8
8
  class GooglePublicDNS < Base
9
9
  #
10
- # Query Google Public DNS
10
+ # @param [Mihari::Models::Artifact] artifact
11
11
  #
12
- # @param [String] name
12
+ # @return [Mihari::Models::Artifact]
13
13
  #
14
- # @return [Mihari::Structs::GooglePublicDNS::Response]
15
- #
16
- def call(name)
17
- client.query_all name
14
+ def call(artifact)
15
+ return if artifact.domain.nil?
16
+
17
+ res = client.query_all(artifact.domain)
18
+
19
+ artifact.tap do |tapped|
20
+ if tapped.dns_records.empty?
21
+ tapped.dns_records = res.answers.map do |answer|
22
+ Models::DnsRecord.new(resource: answer.resource_type, value: answer.data)
23
+ end
24
+ end
25
+ end
18
26
  end
19
27
 
20
28
  class << self
@@ -28,8 +36,21 @@ module Mihari
28
36
 
29
37
  private
30
38
 
39
+ #
40
+ # @param [Mihari::Models::Artifact] artifact
41
+ #
42
+ # @return [Boolean]
43
+ #
44
+ def callable_relationships?(artifact)
45
+ artifact.dns_records.empty?
46
+ end
47
+
48
+ def supported_data_types
49
+ %w[url domain]
50
+ end
51
+
31
52
  def client
32
- Clients::GooglePublicDNS.new(timeout: timeout)
53
+ @client ||= Clients::GooglePublicDNS.new(timeout: timeout)
33
54
  end
34
55
  end
35
56
  end
@@ -7,18 +7,36 @@ module Mihari
7
7
  #
8
8
  class MMDB < Base
9
9
  #
10
- # Query MMDB
10
+ # @param [Mihari::Models::Artifact] artifact
11
11
  #
12
- # @param [String] ip
12
+ def call(artifact)
13
+ res = client.query(artifact.data)
14
+
15
+ artifact.tap do |tapped|
16
+ tapped.autonomous_system ||= Models::AutonomousSystem.new(number: res.asn) if res.asn
17
+ if res.country_code
18
+ tapped.geolocation ||= Models::Geolocation.new(
19
+ country: NormalizeCountry(res.country_code, to: :short),
20
+ country_code: res.country_code
21
+ )
22
+ end
23
+ end
24
+ end
25
+
26
+ private
27
+
28
+ #
29
+ # @param [Mihari::Models::Artifact] artifact
13
30
  #
14
- # @return [Mihari::Structs::MMDB::Response]
31
+ # @return [Boolean]
15
32
  #
16
- def call(ip)
17
- client.query ip
33
+ def callable_relationships?(artifact)
34
+ artifact.geolocation.nil? || artifact.autonomous_system.nil?
18
35
  end
19
- memo_wise :call
20
36
 
21
- private
37
+ def supported_data_types
38
+ %w[ip]
39
+ end
22
40
 
23
41
  def client
24
42
  @client ||= Clients::MMDB.new(timeout: timeout)
@@ -9,17 +9,48 @@ module Mihari
9
9
  #
10
10
  # Query Shodan Internet DB
11
11
  #
12
- # @param [String] ip
12
+ # @param [Mihari::Models::Artifact] artifact
13
13
  #
14
14
  # @return [Mihari::Structs::Shodan::InternetDBResponse, nil]
15
15
  #
16
- def call(ip)
17
- client.query ip
16
+ def call(artifact)
17
+ res = client.query(artifact.data)
18
+
19
+ artifact.tap do |tapped|
20
+ tapped.cpes = (res&.cpes || []).map { |cpe| Models::CPE.new(name: cpe) } if tapped.cpes.empty?
21
+ tapped.ports = (res&.ports || []).map { |port| Models::Port.new(number: port) } if tapped.ports.empty?
22
+ if tapped.reverse_dns_names.empty?
23
+ tapped.reverse_dns_names = (res&.hostnames || []).map do |name|
24
+ Models::ReverseDnsName.new(name: name)
25
+ end
26
+ end
27
+ end
28
+ end
29
+
30
+ #
31
+ # @param [Mihari::Models::Artifact] artifact
32
+ #
33
+ # @return [Boolean]
34
+ #
35
+ def callable?(artifact)
36
+ false unless supported_data_types.include?(artifact.data_type)
18
37
  end
19
- memo_wise :call
20
38
 
21
39
  private
22
40
 
41
+ #
42
+ # @param [Mihari::Models::Artifact] artifact
43
+ #
44
+ # @return [Boolean]
45
+ #
46
+ def callable_relationships?(artifact)
47
+ artifact.cpes.empty? || artifact.ports.empty? || artifact.reverse_dns_names.empty?
48
+ end
49
+
50
+ def supported_data_types
51
+ %w[ip]
52
+ end
53
+
23
54
  def client
24
55
  @client ||= Clients::ShodanInternetDB.new(timeout: timeout)
25
56
  end
@@ -8,46 +8,54 @@ module Mihari
8
8
  # Whois enricher
9
9
  #
10
10
  class Whois < Base
11
+ prepend MemoWise
12
+
13
+ #
14
+ # Query IAIA Whois API
11
15
  #
12
- # @param [Hash, nil] options
16
+ # @param [Mihari::Models::Artifact] artifact
13
17
  #
14
- def initialize(options: nil)
15
- super(options: options)
18
+ def call(artifact)
19
+ return if artifact.domain.nil?
20
+
21
+ domain = PublicSuffix.domain(artifact.domain)
22
+ record = memoized_lookup(domain)
23
+ return if record.parser.available?
24
+
25
+ artifact.whois_record ||= Models::WhoisRecord.new(
26
+ domain: domain,
27
+ created_on: get_created_on(record.parser),
28
+ updated_on: get_updated_on(record.parser),
29
+ expires_on: get_expires_on(record.parser),
30
+ registrar: get_registrar(record.parser),
31
+ contacts: get_contacts(record.parser)
32
+ )
16
33
  end
17
34
 
35
+ private
36
+
18
37
  #
19
- # Query IAIA Whois API
20
- #
21
- # @param [String] domain
38
+ # @param [Mihari::Models::Artifact] artifact
22
39
  #
23
- # @return [Mihari::Models::WhoisRecord, nil]
40
+ # @return [Boolean]
24
41
  #
25
- def call(domain)
26
- memoized_call PublicSuffix.domain(domain)
42
+ def callable_relationships?(artifact)
43
+ artifact.whois_record.nil?
27
44
  end
28
45
 
29
- private
46
+ def supported_data_types
47
+ %w[url domain]
48
+ end
30
49
 
31
50
  #
32
51
  # @param [String] domain
33
52
  #
34
53
  # @return [Mihari::Models::WhoisRecord, nil]
35
54
  #
36
- def memoized_call(domain)
37
- record = whois.lookup(domain)
38
- parser = record.parser
39
- return nil if parser.available?
40
-
41
- Models::WhoisRecord.new(
42
- domain: domain,
43
- created_on: get_created_on(parser),
44
- updated_on: get_updated_on(parser),
45
- expires_on: get_expires_on(parser),
46
- registrar: get_registrar(parser),
47
- contacts: get_contacts(parser)
48
- )
55
+ def memoized_lookup(domain)
56
+ whois.lookup domain
49
57
  end
50
- memo_wise :memoized_call
58
+ memo_wise :memoized_lookup
51
59
 
52
60
  #
53
61
  # @return [::Whois::Client]
@@ -6,6 +6,18 @@ module Mihari
6
6
  # Alert model
7
7
  #
8
8
  class Alert < ActiveRecord::Base
9
+ # @!attribute [r] id
10
+ # @return [Integer, nil]
11
+
12
+ # @!attribute [rw] created_at
13
+ # @return [DateTime]
14
+
15
+ # @!attribute [r] rule
16
+ # @return [Mihari::Models::Rule]
17
+
18
+ # @!attribute [r] artifacts
19
+ # @return [Array<Mihari::Models::Artifact>]
20
+
9
21
  belongs_to :rule
10
22
 
11
23
  has_many :artifacts, dependent: :destroy
@@ -17,6 +17,90 @@ module Mihari
17
17
  # Artifact model
18
18
  #
19
19
  class Artifact < ActiveRecord::Base
20
+ # @!attribute [r] id
21
+ # @return [Integer, nil]
22
+
23
+ # @!attribute [rw] data
24
+ # @return [String]
25
+
26
+ # @!attribute [rw] data_type
27
+ # @return [String]
28
+
29
+ # @!attribute [rw] source
30
+ # @return [String, nil]
31
+
32
+ # @!attribute [rw] query
33
+ # @return [String, nil]
34
+
35
+ # @!attribute [rw] metadata
36
+ # @return [Hash, nil]
37
+
38
+ # @!attribute [rw] created_at
39
+ # @return [DateTime]
40
+
41
+ # @!attribute [r] alert
42
+ # @return [Mihari::Models::Alert]
43
+
44
+ # @!attribute [r] rule
45
+ # @return [Mihari::Models::Rule]
46
+
47
+ # @!attribute [rw] autonomous_system
48
+ # @return [Mihari::Models::AutonomousSystem, nil]
49
+
50
+ # @!attribute [rw] geolocation
51
+ # @return [Mihari::Models::Geolocation, nil]
52
+
53
+ # @!attribute [rw] whois_record
54
+ # @return [Mihari::Models::WhoisRecord, nil]
55
+
56
+ # @!attribute [rw] cpes
57
+ # @return [Array<Mihari::Models::CPE>]
58
+
59
+ # @!attribute [rw] dns_records
60
+ # @return [Array<Mihari::Models::DnsRecord>]
61
+
62
+ # @!attribute [rw] ports
63
+ # @return [Array<Mihari::Models::Port>]
64
+
65
+ # @!attribute [rw] reverse_dns_names
66
+ # @return [Array<Mihari::Models::ReverseDnsName>]
67
+
68
+ # @!attribute [rw] vulnerabilities
69
+ # @return [Array<Mihari::Models::Vulnerability>]
70
+
71
+ # @!attribute [r] alert
72
+ # @return [Mihari::Models::Alert]
73
+
74
+ # @!attribute [r] rule
75
+ # @return [Mihari::Models::Rule]
76
+
77
+ # @!attribute [rw] autonomous_system
78
+ # @return [Mihari::Models::AutonomousSystem, nil]
79
+
80
+ # @!attribute [rw] geolocation
81
+ # @return [Mihari::Models::Geolocation, nil]
82
+
83
+ # @!attribute [rw] whois_record
84
+ # @return [Mihari::Models::WhoisRecord, nil]
85
+
86
+ # @!attribute [rw] cpes
87
+ # @return [Array<Mihari::Models::CPE>]
88
+
89
+ # @!attribute [rw] dns_records
90
+ # @return [Array<Mihari::Models::DnsRecord>]
91
+
92
+ # @!attribute [rw] ports
93
+ # @return [Array<Mihari::Models::Port>]
94
+
95
+ # @!attribute [rw] reverse_dns_names
96
+ # @return [Array<Mihari::Models::ReverseDnsName>]
97
+
98
+ # @!attribute [rw] vulnerabilities
99
+ # @return [Array<Mihari::Models::Vulnerability>]
100
+
101
+ # @!attribute [rw] tags
102
+ # @return [Array<Mihari::Models::Tag>]
103
+
20
104
  belongs_to :alert
21
105
 
22
106
  has_one :autonomous_system, dependent: :destroy
@@ -90,173 +174,24 @@ module Mihari
90
174
  artifact.created_at < decayed_at
91
175
  end
92
176
 
93
- #
94
- # Enrich whois record
95
- #
96
- # @param [Mihari::Enrichers::Whois] enricher
97
- #
98
- def enrich_whois(enricher = Enrichers::Whois.new)
99
- return unless can_enrich_whois?
100
-
101
- self.whois_record = Services::WhoisRecordBuilder.call(domain, enricher: enricher)
102
- end
103
-
104
- #
105
- # Enrich DNS records
106
- #
107
- # @param [Mihari::Enrichers::GooglePublicDNS] enricher
108
- #
109
- def enrich_dns(enricher = Enrichers::GooglePublicDNS.new)
110
- return unless can_enrich_dns?
111
-
112
- self.dns_records = Services::DnsRecordBuilder.call(domain, enricher: enricher)
113
- end
114
-
115
- #
116
- # Enrich reverse DNS names
117
- #
118
- # @param [Mihari::Enrichers::Shodan] enricher
119
- #
120
- def enrich_reverse_dns(enricher = Enrichers::Shodan.new)
121
- return unless can_enrich_reverse_dns?
122
-
123
- self.reverse_dns_names = Services::ReverseDnsNameBuilder.call(data, enricher: enricher)
124
- end
125
-
126
- #
127
- # Enrich geolocation
128
- #
129
- # @param [Mihari::Enrichers::IPInfo] enricher
130
- #
131
- def enrich_geolocation(enricher = Enrichers::MMDB.new)
132
- return unless can_enrich_geolocation?
133
-
134
- self.geolocation = Services::GeolocationBuilder.call(data, enricher: enricher)
135
- end
136
-
137
- #
138
- # Enrich AS
139
- #
140
- # @param [Mihari::Enrichers::IPInfo] enricher
141
- #
142
- def enrich_autonomous_system(enricher = Enrichers::MMDB.new)
143
- return unless can_enrich_autonomous_system?
144
-
145
- self.autonomous_system = Services::AutonomousSystemBuilder.call(data, enricher: enricher)
146
- end
147
-
148
- #
149
- # Enrich ports
150
- #
151
- # @param [Mihari::Enrichers::Shodan] enricher
152
- #
153
- def enrich_ports(enricher = Enrichers::Shodan.new)
154
- return unless can_enrich_ports?
155
-
156
- self.ports = Services::PortBuilder.call(data, enricher: enricher)
157
- end
158
-
159
- #
160
- # Enrich CPEs
161
- #
162
- # @param [Mihari::Enrichers::Shodan] enricher
163
- #
164
- def enrich_cpes(enricher = Enrichers::Shodan.new)
165
- return unless can_enrich_cpes?
166
-
167
- self.cpes = Services::CPEBuilder.call(data, enricher: enricher)
168
- end
169
-
170
- #
171
- # Enrich vulnerabilities
172
- #
173
- # @param [Mihari::Enrichers::Shodan] enricher
174
- #
175
- def enrich_vulnerabilities(enricher = Enrichers::Shodan.new)
176
- return unless can_enrich_vulnerabilities?
177
-
178
- self.vulnerabilities = Services::VulnerabilityBuilder.call(data, enricher: enricher)
177
+ def enrichable?
178
+ !callable_enrichers.empty?
179
179
  end
180
180
 
181
- #
182
- # Enrich all the enrichable relationships of the artifact
183
- #
184
- def enrich_all
185
- enrich_autonomous_system mmdb
186
- enrich_dns
187
- enrich_geolocation mmdb
188
- enrich_reverse_dns shodan
189
- enrich_whois
190
- enrich_ports shodan
191
- enrich_cpes shodan
192
- enrich_vulnerabilities shodan
181
+ def enrich
182
+ callable_enrichers.each { |enricher| enricher.result self }
193
183
  end
194
184
 
195
- ENRICH_METHODS_BY_ENRICHER = {
196
- Enrichers::Whois => %i[
197
- enrich_whois
198
- ],
199
- Enrichers::MMDB => %i[
200
- enrich_autonomous_system
201
- enrich_geolocation
202
- ],
203
- Enrichers::Shodan => %i[
204
- enrich_ports
205
- enrich_cpes
206
- enrich_reverse_dns
207
- enrich_vulnerabilities
208
- ],
209
- Enrichers::GooglePublicDNS => %i[
210
- enrich_dns
211
- ]
212
- }.freeze
213
-
214
185
  #
215
- # Enrich by name of enricher
216
- #
217
- # @param [Mihari::Enrichers::Base] enricher
186
+ # @return [String, nil]
218
187
  #
219
- def enrich_by_enricher(enricher)
220
- methods = ENRICH_METHODS_BY_ENRICHER[enricher.class] || []
221
- methods.each { |method| send(method, enricher) if respond_to?(method) }
222
- end
223
-
224
- def can_enrich_whois?
225
- %w[domain url].include?(data_type) && whois_record.nil?
226
- end
227
-
228
- def can_enrich_dns?
229
- %w[domain url].include?(data_type) && dns_records.empty?
230
- end
231
-
232
- def can_enrich_reverse_dns?
233
- data_type == "ip" && reverse_dns_names.empty?
234
- end
235
-
236
- def can_enrich_geolocation?
237
- data_type == "ip" && geolocation.nil?
238
- end
239
-
240
- def can_enrich_autonomous_system?
241
- data_type == "ip" && autonomous_system.nil?
242
- end
243
-
244
- def can_enrich_ports?
245
- data_type == "ip" && ports.empty?
246
- end
247
-
248
- def can_enrich_cpes?
249
- data_type == "ip" && cpes.empty?
250
- end
251
-
252
- def can_enrich_vulnerabilities?
253
- data_type == "ip" && vulnerabilities.empty?
254
- end
255
-
256
- def enrichable?
257
- enrich_methods = methods.map(&:to_s).select { |method| method.start_with?("can_enrich_") }
258
- enrich_methods.map(&:to_sym).any? do |method|
259
- send(method) if respond_to?(method)
188
+ def domain
189
+ case data_type
190
+ when "domain"
191
+ data
192
+ when "url"
193
+ host = Addressable::URI.parse(data).host
194
+ (DataType.type(host) == "ip") ? nil : host
260
195
  end
261
196
  end
262
197
 
@@ -272,6 +207,15 @@ module Mihari
272
207
 
273
208
  private
274
209
 
210
+ #
211
+ # @return [Array<Mihari::Enrichers::Base>]
212
+ #
213
+ def callable_enrichers
214
+ @callable_enrichers ||= Mihari.enrichers.map(&:new).select do |enricher|
215
+ enricher.callable?(self)
216
+ end
217
+ end
218
+
275
219
  def set_data_type
276
220
  self.data_type = DataType.type(data)
277
221
  end
@@ -279,26 +223,6 @@ module Mihari
279
223
  def set_rule_id
280
224
  @set_rule_id ||= nil
281
225
  end
282
-
283
- def mmdb
284
- @mmdb ||= Enrichers::MMDB.new
285
- end
286
-
287
- def shodan
288
- @shodan ||= Enrichers::Shodan.new
289
- end
290
-
291
- #
292
- # @return [String, nil]
293
- #
294
- def domain
295
- case data_type
296
- when "domain"
297
- data
298
- when "url"
299
- Addressable::URI.parse(data).host
300
- end
301
- end
302
226
  end
303
227
  end
304
228
  end