mihari 5.2.1 → 5.2.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (63) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +2 -0
  3. data/lib/mihari/analyzers/base.rb +20 -115
  4. data/lib/mihari/analyzers/binaryedge.rb +0 -1
  5. data/lib/mihari/analyzers/censys.rb +26 -3
  6. data/lib/mihari/analyzers/circl.rb +1 -1
  7. data/lib/mihari/analyzers/onyphe.rb +1 -1
  8. data/lib/mihari/analyzers/passivetotal.rb +1 -1
  9. data/lib/mihari/analyzers/rule.rb +122 -75
  10. data/lib/mihari/analyzers/shodan.rb +1 -1
  11. data/lib/mihari/analyzers/urlscan.rb +6 -9
  12. data/lib/mihari/analyzers/virustotal_intelligence.rb +1 -6
  13. data/lib/mihari/cli/main.rb +2 -2
  14. data/lib/mihari/clients/base.rb +1 -1
  15. data/lib/mihari/commands/database.rb +12 -11
  16. data/lib/mihari/commands/rule.rb +47 -45
  17. data/lib/mihari/commands/search.rb +88 -0
  18. data/lib/mihari/commands/version.rb +8 -6
  19. data/lib/mihari/commands/web.rb +26 -23
  20. data/lib/mihari/emitters/base.rb +14 -1
  21. data/lib/mihari/emitters/database.rb +3 -10
  22. data/lib/mihari/emitters/misp.rb +16 -5
  23. data/lib/mihari/emitters/slack.rb +13 -15
  24. data/lib/mihari/emitters/the_hive.rb +17 -19
  25. data/lib/mihari/emitters/webhook.rb +23 -23
  26. data/lib/mihari/enrichers/whois.rb +1 -0
  27. data/lib/mihari/feed/parser.rb +1 -0
  28. data/lib/mihari/feed/reader.rb +29 -14
  29. data/lib/mihari/mixins/configurable.rb +13 -4
  30. data/lib/mihari/mixins/error_notification.rb +0 -2
  31. data/lib/mihari/models/artifact.rb +1 -1
  32. data/lib/mihari/schemas/rule.rb +2 -17
  33. data/lib/mihari/structs/censys.rb +226 -56
  34. data/lib/mihari/structs/config.rb +48 -18
  35. data/lib/mihari/structs/google_public_dns.rb +56 -14
  36. data/lib/mihari/structs/greynoise.rb +122 -29
  37. data/lib/mihari/structs/ipinfo.rb +40 -0
  38. data/lib/mihari/structs/onyphe.rb +112 -26
  39. data/lib/mihari/structs/rule.rb +4 -2
  40. data/lib/mihari/structs/shodan.rb +189 -47
  41. data/lib/mihari/structs/urlscan.rb +123 -20
  42. data/lib/mihari/structs/virustotal_intelligence.rb +129 -26
  43. data/lib/mihari/type_checker.rb +10 -8
  44. data/lib/mihari/version.rb +1 -1
  45. data/lib/mihari.rb +1 -0
  46. data/mihari.gemspec +11 -10
  47. metadata +35 -36
  48. data/.github/ISSUE_TEMPLATE/bug_report.md +0 -43
  49. data/.github/ISSUE_TEMPLATE/feature_request.md +0 -15
  50. data/.github/workflows/test.yml +0 -90
  51. data/config/pre_commit.yml +0 -3
  52. data/docker/Dockerfile +0 -14
  53. data/examples/ipinfo_hosted_domains.rb +0 -45
  54. data/images/Tines-Full_Logo-Tines_Black.png +0 -0
  55. data/images/alert.png +0 -0
  56. data/images/logo.png +0 -0
  57. data/images/misp.png +0 -0
  58. data/images/overview.jpg +0 -0
  59. data/images/slack.png +0 -0
  60. data/images/tines.png +0 -0
  61. data/images/web_alerts.png +0 -0
  62. data/images/web_config.png +0 -0
  63. data/lib/mihari/commands/searcher.rb +0 -61
@@ -4,14 +4,23 @@ module Mihari
4
4
  module Mixins
5
5
  module Configurable
6
6
  #
7
- # Check whether it is configured or not
7
+ # Check whether there are configuration key-values or not
8
8
  #
9
9
  # @return [Boolean]
10
10
  #
11
- def configured?
11
+ def configuration_keys?
12
12
  return true if configuration_keys.empty?
13
13
 
14
- configuration_keys.all? { |key| Mihari.config.send(key) } || api_key?
14
+ configuration_keys.all? { |key| Mihari.config.send(key) }
15
+ end
16
+
17
+ #
18
+ # Check whether it is configured or not
19
+ #
20
+ # @return [Boolean]
21
+ #
22
+ def configured?
23
+ configuration_keys? || api_key?
15
24
  end
16
25
 
17
26
  #
@@ -32,7 +41,7 @@ module Mihari
32
41
  #
33
42
  # Configuration keys
34
43
  #
35
- # @return [Array<String>] A list of cofiguration keys
44
+ # @return [Array<String>] A list of configuration keys
36
45
  #
37
46
  def configuration_keys
38
47
  []
@@ -6,8 +6,6 @@ module Mihari
6
6
  #
7
7
  # Send an exception notification if there is any error in a block
8
8
  #
9
- # @return [Nil]
10
- #
11
9
  def with_error_notification
12
10
  yield
13
11
  rescue StandardError => e
@@ -60,7 +60,7 @@ module Mihari
60
60
  ).order(created_at: :desc).first
61
61
  return true if artifact.nil?
62
62
 
63
- # check whetehr the artifact is decayed or not
63
+ # check whether the artifact is decayed or not
64
64
  return false if artifact_lifetime.nil?
65
65
 
66
66
  # use the current UTC time if base_time is not given (for testing)
@@ -25,30 +25,15 @@ module Mihari
25
25
  AnalyzerWithoutAPIKey | AnalyzerWithAPIKey | Censys | CIRCL | PassiveTotal | ZoomEye | Urlscan | Crtsh | Feed
26
26
  end
27
27
 
28
- optional(:emitters).value(:array).each { Database | MISP | TheHive | Slack | Webhook }
28
+ optional(:emitters).value(:array).each { Database | MISP | TheHive | Slack | Webhook }.default(DEFAULT_EMITTERS)
29
29
 
30
- optional(:enrichers).value(:array).each(Enricher)
30
+ optional(:enrichers).value(:array).each(Enricher).default(DEFAULT_ENRICHERS)
31
31
 
32
32
  optional(:data_types).value(array[Types::DataTypes]).default(DEFAULT_DATA_TYPES)
33
33
  optional(:falsepositives).value(array[:string]).default([])
34
34
 
35
35
  optional(:artifact_lifetime).value(:integer)
36
36
  optional(:artifact_ttl).value(:integer)
37
-
38
- before(:key_coercer) do |result|
39
- # it looks like that dry-schema v1.9.1 has an issue with setting an array of schemas as a default value
40
- # e.g. optional(:emitters).value(:array).each { Emitter | HTTP }.default(DEFAULT_EMITTERS) does not work well
41
- # so let's do a dirty hack...
42
- h = result.to_h
43
-
44
- emitters = h[:emitters]
45
- h[:emitters] = emitters || DEFAULT_EMITTERS
46
-
47
- enrichers = h[:enrichers]
48
- h[:enrichers] = enrichers || DEFAULT_ENRICHERS
49
-
50
- h
51
- end
52
37
  end
53
38
 
54
39
  class RuleContract < Dry::Validation::Contract
@@ -8,6 +8,13 @@ module Mihari
8
8
 
9
9
  attribute :asn, Types::Int
10
10
 
11
+ #
12
+ # @return [Integer]
13
+ #
14
+ def asn
15
+ attributes[:asn]
16
+ end
17
+
11
18
  #
12
19
  # @return [Mihari::AutonomousSystem]
13
20
  #
@@ -15,11 +22,18 @@ module Mihari
15
22
  Mihari::AutonomousSystem.new(asn: normalize_asn(asn))
16
23
  end
17
24
 
18
- def self.from_dynamic!(d)
19
- d = Types::Hash[d]
20
- new(
21
- asn: d.fetch("asn")
22
- )
25
+ class << self
26
+ #
27
+ # @param [Hash] d
28
+ #
29
+ # @return [AutonomousSystem]
30
+ #
31
+ def from_dynamic!(d)
32
+ d = Types::Hash[d]
33
+ new(
34
+ asn: d.fetch("asn")
35
+ )
36
+ end
23
37
  end
24
38
  end
25
39
 
@@ -27,6 +41,20 @@ module Mihari
27
41
  attribute :country, Types::String.optional
28
42
  attribute :country_code, Types::String.optional
29
43
 
44
+ #
45
+ # @return [String, nil]
46
+ #
47
+ def country
48
+ attributes[:country]
49
+ end
50
+
51
+ #
52
+ # @return [String, nil]
53
+ #
54
+ def country_code
55
+ attributes[:country_code]
56
+ end
57
+
30
58
  #
31
59
  # @return [Mihari::Geolocation] <description>
32
60
  #
@@ -41,18 +69,32 @@ module Mihari
41
69
  )
42
70
  end
43
71
 
44
- def self.from_dynamic!(d)
45
- d = Types::Hash[d]
46
- new(
47
- country: d["country"],
48
- country_code: d["country_code"]
49
- )
72
+ class << self
73
+ #
74
+ # @param [Hash] d
75
+ #
76
+ # @return [Location]
77
+ #
78
+ def from_dynamic!(d)
79
+ d = Types::Hash[d]
80
+ new(
81
+ country: d["country"],
82
+ country_code: d["country_code"]
83
+ )
84
+ end
50
85
  end
51
86
  end
52
87
 
53
88
  class Service < Dry::Struct
54
89
  attribute :port, Types::Integer
55
90
 
91
+ #
92
+ # @return [Integer]
93
+ #
94
+ def port
95
+ attributes[:port]
96
+ end
97
+
56
98
  #
57
99
  # @return [Mihari::Port]
58
100
  #
@@ -60,11 +102,18 @@ module Mihari
60
102
  Port.new(port: port)
61
103
  end
62
104
 
63
- def self.from_dynamic!(d)
64
- d = Types::Hash[d]
65
- new(
66
- port: d.fetch("port")
67
- )
105
+ class << self
106
+ #
107
+ # @param [Hash] d
108
+ #
109
+ # @return [Service]
110
+ #
111
+ def from_dynamic!(d)
112
+ d = Types::Hash[d]
113
+ new(
114
+ port: d.fetch("port")
115
+ )
116
+ end
68
117
  end
69
118
  end
70
119
 
@@ -75,6 +124,41 @@ module Mihari
75
124
  attribute :metadata, Types::Hash
76
125
  attribute :services, Types.Array(Service)
77
126
 
127
+ #
128
+ # @return [String]
129
+ #
130
+ def ip
131
+ attributes[:ip]
132
+ end
133
+
134
+ #
135
+ # @return [Location]
136
+ #
137
+ def location
138
+ attributes[:location]
139
+ end
140
+
141
+ #
142
+ # @return [AutonomousSystem]
143
+ #
144
+ def autonomous_system
145
+ attributes[:autonomous_system]
146
+ end
147
+
148
+ #
149
+ # @return [Hash]
150
+ #
151
+ def metadata
152
+ attributes[:metadata]
153
+ end
154
+
155
+ #
156
+ # @return [Array<Service>]
157
+ #
158
+ def services
159
+ attributes[:services]
160
+ end
161
+
78
162
  #
79
163
  # @return [Array<Mihari::Port>]
80
164
  #
@@ -82,15 +166,12 @@ module Mihari
82
166
  services.map(&:to_port)
83
167
  end
84
168
 
85
- #
86
- # @param [String] source
87
169
  #
88
170
  # @return [Mihari::Artifact]
89
171
  #
90
- def to_artifact(source = "Censys")
172
+ def to_artifact
91
173
  Artifact.new(
92
174
  data: ip,
93
- source: source,
94
175
  metadata: metadata,
95
176
  autonomous_system: autonomous_system.to_as,
96
177
  geolocation: location.to_geolocation,
@@ -98,28 +179,56 @@ module Mihari
98
179
  )
99
180
  end
100
181
 
101
- def self.from_dynamic!(d)
102
- d = Types::Hash[d]
103
- new(
104
- ip: d.fetch("ip"),
105
- location: Location.from_dynamic!(d.fetch("location")),
106
- autonomous_system: AutonomousSystem.from_dynamic!(d.fetch("autonomous_system")),
107
- metadata: d,
108
- services: d.fetch("services", []).map { |x| Service.from_dynamic!(x) }
109
- )
182
+ class << self
183
+ #
184
+ # @param [Hash] d
185
+ #
186
+ # @return [Hit]
187
+ #
188
+ def from_dynamic!(d)
189
+ d = Types::Hash[d]
190
+ new(
191
+ ip: d.fetch("ip"),
192
+ location: Location.from_dynamic!(d.fetch("location")),
193
+ autonomous_system: AutonomousSystem.from_dynamic!(d.fetch("autonomous_system")),
194
+ metadata: d,
195
+ services: d.fetch("services", []).map { |x| Service.from_dynamic!(x) }
196
+ )
197
+ end
110
198
  end
111
199
  end
112
200
 
113
201
  class Links < Dry::Struct
114
- attribute :next, Types::String
115
- attribute :prev, Types::String
116
-
117
- def self.from_dynamic!(d)
118
- d = Types::Hash[d]
119
- new(
120
- next: d.fetch("next"),
121
- prev: d.fetch("prev")
122
- )
202
+ attribute :next, Types::String.optional
203
+ attribute :prev, Types::String.optional
204
+
205
+ #
206
+ # @return [String, nil]
207
+ #
208
+ def next
209
+ attributes[:next]
210
+ end
211
+
212
+ #
213
+ # @return [String, nil]
214
+ #
215
+ def prev
216
+ attributes[:prev]
217
+ end
218
+
219
+ class << self
220
+ #
221
+ # @param [Hash] d
222
+ #
223
+ # @return [Links]
224
+ #
225
+ def from_dynamic!(d)
226
+ d = Types::Hash[d]
227
+ new(
228
+ next: d["next"],
229
+ prev: d["prev"]
230
+ )
231
+ end
123
232
  end
124
233
  end
125
234
 
@@ -130,22 +239,55 @@ module Mihari
130
239
  attribute :links, Links
131
240
 
132
241
  #
133
- # @param [String] source
242
+ # @return [String]
243
+ #
244
+ def query
245
+ attributes[:query]
246
+ end
247
+
248
+ #
249
+ # @return [Integer]
250
+ #
251
+ def total
252
+ attributes[:total]
253
+ end
254
+
255
+ #
256
+ # @return [Array<Hit>]
257
+ #
258
+ def hits
259
+ attributes[:hits]
260
+ end
261
+
262
+ #
263
+ # @return [Links]
264
+ #
265
+ def links
266
+ attributes[:links]
267
+ end
268
+
134
269
  #
135
270
  # @return [Array<Mihari::Artifact>]
136
271
  #
137
- def to_artifacts(source = "Censys")
138
- hits.map { |hit| hit.to_artifact(source) }
272
+ def to_artifacts
273
+ hits.map(&:to_artifact)
139
274
  end
140
275
 
141
- def self.from_dynamic!(d)
142
- d = Types::Hash[d]
143
- new(
144
- query: d.fetch("query"),
145
- total: d.fetch("total"),
146
- hits: d.fetch("hits", []).map { |x| Hit.from_dynamic!(x) },
147
- links: Links.from_dynamic!(d.fetch("links"))
148
- )
276
+ class << self
277
+ #
278
+ # @param [Hash] d
279
+ #
280
+ # @return [Result]
281
+ #
282
+ def from_dynamic!(d)
283
+ d = Types::Hash[d]
284
+ new(
285
+ query: d.fetch("query"),
286
+ total: d.fetch("total"),
287
+ hits: d.fetch("hits", []).map { |x| Hit.from_dynamic!(x) },
288
+ links: Links.from_dynamic!(d.fetch("links"))
289
+ )
290
+ end
149
291
  end
150
292
  end
151
293
 
@@ -154,13 +296,41 @@ module Mihari
154
296
  attribute :status, Types::String
155
297
  attribute :result, Result
156
298
 
157
- def self.from_dynamic!(d)
158
- d = Types::Hash[d]
159
- new(
160
- code: d.fetch("code"),
161
- status: d.fetch("status"),
162
- result: Result.from_dynamic!(d.fetch("result"))
163
- )
299
+ #
300
+ # @return [Integer]
301
+ #
302
+ def code
303
+ attributes[:code]
304
+ end
305
+
306
+ #
307
+ # @return [String]
308
+ #
309
+ def status
310
+ attributes[:status]
311
+ end
312
+
313
+ #
314
+ # @return [Result]
315
+ #
316
+ def result
317
+ attributes[:result]
318
+ end
319
+
320
+ class << self
321
+ #
322
+ # @param [Hash] d
323
+ #
324
+ # @return [Response]
325
+ #
326
+ def from_dynamic!(d)
327
+ d = Types::Hash[d]
328
+ new(
329
+ code: d.fetch("code"),
330
+ status: d.fetch("status"),
331
+ result: Result.from_dynamic!(d.fetch("result"))
332
+ )
333
+ end
164
334
  end
165
335
  end
166
336
  end
@@ -9,31 +9,61 @@ module Mihari
9
9
  attribute :values, Types.Array(Types::Hash).optional
10
10
 
11
11
  #
12
- # @param [Class<Mihari::Analyzers::Base>, Class<Mihari::Emitters::Base>] klass
12
+ # @return [String]
13
13
  #
14
- # @return [Mihari::Structs::Config, nil] config
14
+ def name
15
+ attributes[:name]
16
+ end
17
+
18
+ #
19
+ # @return [String]
20
+ #
21
+ def type
22
+ attributes[:type]
23
+ end
24
+
25
+ #
26
+ # @return [Boolean]
15
27
  #
16
- def self.from_class(klass)
17
- return nil if klass == Mihari::Analyzers::Rule
28
+ def is_configured
29
+ attributes[:is_configured]
30
+ end
31
+
32
+ #
33
+ # @return [Array<Hash>]
34
+ #
35
+ def values
36
+ attributes[:values]
37
+ end
38
+
39
+ class << self
40
+ #
41
+ # @param [Class<Mihari::Analyzers::Base>, Class<Mihari::Emitters::Base>] klass
42
+ #
43
+ # @return [Mihari::Structs::Config, nil] config
44
+ #
45
+ def from_class(klass)
46
+ return nil if klass == Mihari::Analyzers::Rule
18
47
 
19
- name = klass.to_s.split("::").last.to_s
48
+ name = klass.to_s.split("::").last.to_s
20
49
 
21
- is_analyzer = klass.ancestors.include?(Mihari::Analyzers::Base)
22
- is_emitter = klass.ancestors.include?(Mihari::Emitters::Base)
23
- is_enricher = klass.ancestors.include?(Mihari::Enrichers::Base)
50
+ is_analyzer = klass.ancestors.include?(Mihari::Analyzers::Base)
51
+ is_emitter = klass.ancestors.include?(Mihari::Emitters::Base)
52
+ is_enricher = klass.ancestors.include?(Mihari::Enrichers::Base)
24
53
 
25
- type = "Analyzer"
26
- type = "Emitter" if is_emitter
27
- type = "Enricher" if is_enricher
54
+ type = "Analyzer"
55
+ type = "Emitter" if is_emitter
56
+ type = "Enricher" if is_enricher
28
57
 
29
- begin
30
- instance = is_analyzer ? klass.new("dummy") : klass.new
31
- is_configured = instance.configured?
32
- values = instance.configuration_values
58
+ begin
59
+ instance = is_analyzer ? klass.new("dummy") : klass.new(artifacts: [], rule: nil)
60
+ is_configured = instance.configured?
61
+ values = instance.configuration_values
33
62
 
34
- new(name: name, values: values, is_configured: is_configured, type: type)
35
- rescue ArgumentError => _e
36
- nil
63
+ new(name: name, values: values, is_configured: is_configured, type: type)
64
+ rescue ArgumentError => _e
65
+ nil
66
+ end
37
67
  end
38
68
  end
39
69
  end
@@ -9,32 +9,74 @@ module Mihari
9
9
  5 => "CNAME",
10
10
  16 => "TXT",
11
11
  28 => "AAAA"
12
- }
12
+ }.freeze
13
13
 
14
14
  class Answer < Dry::Struct
15
15
  attribute :name, Types::String
16
16
  attribute :data, Types::String
17
17
  attribute :resource_type, Types::String
18
18
 
19
- def self.from_dynamic!(d)
20
- d = Types::Hash[d]
21
- resource_type = INT_TYPE_TO_TYPE[d.fetch("type")]
22
- new(
23
- name: d.fetch("name"),
24
- data: d.fetch("data"),
25
- resource_type: resource_type
26
- )
19
+ #
20
+ # @return [String]
21
+ #
22
+ def name
23
+ attributes[:name]
24
+ end
25
+
26
+ #
27
+ # @return [String]
28
+ #
29
+ def data
30
+ attributes[:data]
31
+ end
32
+
33
+ #
34
+ # @return [String]
35
+ #
36
+ def resource_type
37
+ attributes[:resource_type]
38
+ end
39
+
40
+ class << self
41
+ #
42
+ # @param [Hash] d
43
+ #
44
+ # @return [Answer]
45
+ #
46
+ def from_dynamic!(d)
47
+ d = Types::Hash[d]
48
+ resource_type = INT_TYPE_TO_TYPE[d.fetch("type")]
49
+ new(
50
+ name: d.fetch("name"),
51
+ data: d.fetch("data"),
52
+ resource_type: resource_type
53
+ )
54
+ end
27
55
  end
28
56
  end
29
57
 
30
58
  class Response < Dry::Struct
31
59
  attribute :answers, Types.Array(Answer)
32
60
 
33
- def self.from_dynamic!(d)
34
- d = Types::Hash[d]
35
- new(
36
- answers: d.fetch("Answer", []).map { |x| Answer.from_dynamic!(x) }
37
- )
61
+ #
62
+ # @return [Array<Answer>]
63
+ #
64
+ def answers
65
+ attributes[:answers]
66
+ end
67
+
68
+ class << self
69
+ #
70
+ # @param [Hash] d
71
+ #
72
+ # @return [Response]
73
+ #
74
+ def from_dynamic!(d)
75
+ d = Types::Hash[d]
76
+ new(
77
+ answers: d.fetch("Answer", []).map { |x| Answer.from_dynamic!(x) }
78
+ )
79
+ end
38
80
  end
39
81
  end
40
82
  end