mihari 5.2.1 → 5.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +2 -0
  3. data/lib/mihari/analyzers/base.rb +20 -115
  4. data/lib/mihari/analyzers/binaryedge.rb +0 -1
  5. data/lib/mihari/analyzers/censys.rb +26 -3
  6. data/lib/mihari/analyzers/circl.rb +1 -1
  7. data/lib/mihari/analyzers/onyphe.rb +1 -1
  8. data/lib/mihari/analyzers/passivetotal.rb +1 -1
  9. data/lib/mihari/analyzers/rule.rb +122 -75
  10. data/lib/mihari/analyzers/shodan.rb +1 -1
  11. data/lib/mihari/analyzers/urlscan.rb +6 -9
  12. data/lib/mihari/analyzers/virustotal_intelligence.rb +1 -6
  13. data/lib/mihari/cli/main.rb +2 -2
  14. data/lib/mihari/clients/base.rb +1 -1
  15. data/lib/mihari/commands/database.rb +12 -11
  16. data/lib/mihari/commands/rule.rb +47 -45
  17. data/lib/mihari/commands/search.rb +88 -0
  18. data/lib/mihari/commands/version.rb +8 -6
  19. data/lib/mihari/commands/web.rb +26 -23
  20. data/lib/mihari/emitters/base.rb +14 -1
  21. data/lib/mihari/emitters/database.rb +3 -10
  22. data/lib/mihari/emitters/misp.rb +16 -5
  23. data/lib/mihari/emitters/slack.rb +13 -15
  24. data/lib/mihari/emitters/the_hive.rb +17 -19
  25. data/lib/mihari/emitters/webhook.rb +23 -23
  26. data/lib/mihari/enrichers/whois.rb +1 -0
  27. data/lib/mihari/feed/parser.rb +1 -0
  28. data/lib/mihari/feed/reader.rb +29 -14
  29. data/lib/mihari/mixins/configurable.rb +13 -4
  30. data/lib/mihari/mixins/error_notification.rb +0 -2
  31. data/lib/mihari/models/artifact.rb +1 -1
  32. data/lib/mihari/schemas/rule.rb +2 -17
  33. data/lib/mihari/structs/censys.rb +226 -56
  34. data/lib/mihari/structs/config.rb +48 -18
  35. data/lib/mihari/structs/google_public_dns.rb +56 -14
  36. data/lib/mihari/structs/greynoise.rb +122 -29
  37. data/lib/mihari/structs/ipinfo.rb +40 -0
  38. data/lib/mihari/structs/onyphe.rb +112 -26
  39. data/lib/mihari/structs/rule.rb +4 -2
  40. data/lib/mihari/structs/shodan.rb +189 -47
  41. data/lib/mihari/structs/urlscan.rb +123 -20
  42. data/lib/mihari/structs/virustotal_intelligence.rb +129 -26
  43. data/lib/mihari/type_checker.rb +10 -8
  44. data/lib/mihari/version.rb +1 -1
  45. data/lib/mihari.rb +1 -0
  46. data/mihari.gemspec +11 -10
  47. metadata +35 -36
  48. data/.github/ISSUE_TEMPLATE/bug_report.md +0 -43
  49. data/.github/ISSUE_TEMPLATE/feature_request.md +0 -15
  50. data/.github/workflows/test.yml +0 -90
  51. data/config/pre_commit.yml +0 -3
  52. data/docker/Dockerfile +0 -14
  53. data/examples/ipinfo_hosted_domains.rb +0 -45
  54. data/images/Tines-Full_Logo-Tines_Black.png +0 -0
  55. data/images/alert.png +0 -0
  56. data/images/logo.png +0 -0
  57. data/images/misp.png +0 -0
  58. data/images/overview.jpg +0 -0
  59. data/images/slack.png +0 -0
  60. data/images/tines.png +0 -0
  61. data/images/web_alerts.png +0 -0
  62. data/images/web_config.png +0 -0
  63. data/lib/mihari/commands/searcher.rb +0 -61
@@ -4,14 +4,23 @@ module Mihari
4
4
  module Mixins
5
5
  module Configurable
6
6
  #
7
- # Check whether it is configured or not
7
+ # Check whether there are configuration key-values or not
8
8
  #
9
9
  # @return [Boolean]
10
10
  #
11
- def configured?
11
+ def configuration_keys?
12
12
  return true if configuration_keys.empty?
13
13
 
14
- configuration_keys.all? { |key| Mihari.config.send(key) } || api_key?
14
+ configuration_keys.all? { |key| Mihari.config.send(key) }
15
+ end
16
+
17
+ #
18
+ # Check whether it is configured or not
19
+ #
20
+ # @return [Boolean]
21
+ #
22
+ def configured?
23
+ configuration_keys? || api_key?
15
24
  end
16
25
 
17
26
  #
@@ -32,7 +41,7 @@ module Mihari
32
41
  #
33
42
  # Configuration keys
34
43
  #
35
- # @return [Array<String>] A list of cofiguration keys
44
+ # @return [Array<String>] A list of configuration keys
36
45
  #
37
46
  def configuration_keys
38
47
  []
@@ -6,8 +6,6 @@ module Mihari
6
6
  #
7
7
  # Send an exception notification if there is any error in a block
8
8
  #
9
- # @return [Nil]
10
- #
11
9
  def with_error_notification
12
10
  yield
13
11
  rescue StandardError => e
@@ -60,7 +60,7 @@ module Mihari
60
60
  ).order(created_at: :desc).first
61
61
  return true if artifact.nil?
62
62
 
63
- # check whetehr the artifact is decayed or not
63
+ # check whether the artifact is decayed or not
64
64
  return false if artifact_lifetime.nil?
65
65
 
66
66
  # use the current UTC time if base_time is not given (for testing)
@@ -25,30 +25,15 @@ module Mihari
25
25
  AnalyzerWithoutAPIKey | AnalyzerWithAPIKey | Censys | CIRCL | PassiveTotal | ZoomEye | Urlscan | Crtsh | Feed
26
26
  end
27
27
 
28
- optional(:emitters).value(:array).each { Database | MISP | TheHive | Slack | Webhook }
28
+ optional(:emitters).value(:array).each { Database | MISP | TheHive | Slack | Webhook }.default(DEFAULT_EMITTERS)
29
29
 
30
- optional(:enrichers).value(:array).each(Enricher)
30
+ optional(:enrichers).value(:array).each(Enricher).default(DEFAULT_ENRICHERS)
31
31
 
32
32
  optional(:data_types).value(array[Types::DataTypes]).default(DEFAULT_DATA_TYPES)
33
33
  optional(:falsepositives).value(array[:string]).default([])
34
34
 
35
35
  optional(:artifact_lifetime).value(:integer)
36
36
  optional(:artifact_ttl).value(:integer)
37
-
38
- before(:key_coercer) do |result|
39
- # it looks like that dry-schema v1.9.1 has an issue with setting an array of schemas as a default value
40
- # e.g. optional(:emitters).value(:array).each { Emitter | HTTP }.default(DEFAULT_EMITTERS) does not work well
41
- # so let's do a dirty hack...
42
- h = result.to_h
43
-
44
- emitters = h[:emitters]
45
- h[:emitters] = emitters || DEFAULT_EMITTERS
46
-
47
- enrichers = h[:enrichers]
48
- h[:enrichers] = enrichers || DEFAULT_ENRICHERS
49
-
50
- h
51
- end
52
37
  end
53
38
 
54
39
  class RuleContract < Dry::Validation::Contract
@@ -8,6 +8,13 @@ module Mihari
8
8
 
9
9
  attribute :asn, Types::Int
10
10
 
11
+ #
12
+ # @return [Integer]
13
+ #
14
+ def asn
15
+ attributes[:asn]
16
+ end
17
+
11
18
  #
12
19
  # @return [Mihari::AutonomousSystem]
13
20
  #
@@ -15,11 +22,18 @@ module Mihari
15
22
  Mihari::AutonomousSystem.new(asn: normalize_asn(asn))
16
23
  end
17
24
 
18
- def self.from_dynamic!(d)
19
- d = Types::Hash[d]
20
- new(
21
- asn: d.fetch("asn")
22
- )
25
+ class << self
26
+ #
27
+ # @param [Hash] d
28
+ #
29
+ # @return [AutonomousSystem]
30
+ #
31
+ def from_dynamic!(d)
32
+ d = Types::Hash[d]
33
+ new(
34
+ asn: d.fetch("asn")
35
+ )
36
+ end
23
37
  end
24
38
  end
25
39
 
@@ -27,6 +41,20 @@ module Mihari
27
41
  attribute :country, Types::String.optional
28
42
  attribute :country_code, Types::String.optional
29
43
 
44
+ #
45
+ # @return [String, nil]
46
+ #
47
+ def country
48
+ attributes[:country]
49
+ end
50
+
51
+ #
52
+ # @return [String, nil]
53
+ #
54
+ def country_code
55
+ attributes[:country_code]
56
+ end
57
+
30
58
  #
31
59
  # @return [Mihari::Geolocation] <description>
32
60
  #
@@ -41,18 +69,32 @@ module Mihari
41
69
  )
42
70
  end
43
71
 
44
- def self.from_dynamic!(d)
45
- d = Types::Hash[d]
46
- new(
47
- country: d["country"],
48
- country_code: d["country_code"]
49
- )
72
+ class << self
73
+ #
74
+ # @param [Hash] d
75
+ #
76
+ # @return [Location]
77
+ #
78
+ def from_dynamic!(d)
79
+ d = Types::Hash[d]
80
+ new(
81
+ country: d["country"],
82
+ country_code: d["country_code"]
83
+ )
84
+ end
50
85
  end
51
86
  end
52
87
 
53
88
  class Service < Dry::Struct
54
89
  attribute :port, Types::Integer
55
90
 
91
+ #
92
+ # @return [Integer]
93
+ #
94
+ def port
95
+ attributes[:port]
96
+ end
97
+
56
98
  #
57
99
  # @return [Mihari::Port]
58
100
  #
@@ -60,11 +102,18 @@ module Mihari
60
102
  Port.new(port: port)
61
103
  end
62
104
 
63
- def self.from_dynamic!(d)
64
- d = Types::Hash[d]
65
- new(
66
- port: d.fetch("port")
67
- )
105
+ class << self
106
+ #
107
+ # @param [Hash] d
108
+ #
109
+ # @return [Service]
110
+ #
111
+ def from_dynamic!(d)
112
+ d = Types::Hash[d]
113
+ new(
114
+ port: d.fetch("port")
115
+ )
116
+ end
68
117
  end
69
118
  end
70
119
 
@@ -75,6 +124,41 @@ module Mihari
75
124
  attribute :metadata, Types::Hash
76
125
  attribute :services, Types.Array(Service)
77
126
 
127
+ #
128
+ # @return [String]
129
+ #
130
+ def ip
131
+ attributes[:ip]
132
+ end
133
+
134
+ #
135
+ # @return [Location]
136
+ #
137
+ def location
138
+ attributes[:location]
139
+ end
140
+
141
+ #
142
+ # @return [AutonomousSystem]
143
+ #
144
+ def autonomous_system
145
+ attributes[:autonomous_system]
146
+ end
147
+
148
+ #
149
+ # @return [Hash]
150
+ #
151
+ def metadata
152
+ attributes[:metadata]
153
+ end
154
+
155
+ #
156
+ # @return [Array<Service>]
157
+ #
158
+ def services
159
+ attributes[:services]
160
+ end
161
+
78
162
  #
79
163
  # @return [Array<Mihari::Port>]
80
164
  #
@@ -82,15 +166,12 @@ module Mihari
82
166
  services.map(&:to_port)
83
167
  end
84
168
 
85
- #
86
- # @param [String] source
87
169
  #
88
170
  # @return [Mihari::Artifact]
89
171
  #
90
- def to_artifact(source = "Censys")
172
+ def to_artifact
91
173
  Artifact.new(
92
174
  data: ip,
93
- source: source,
94
175
  metadata: metadata,
95
176
  autonomous_system: autonomous_system.to_as,
96
177
  geolocation: location.to_geolocation,
@@ -98,28 +179,56 @@ module Mihari
98
179
  )
99
180
  end
100
181
 
101
- def self.from_dynamic!(d)
102
- d = Types::Hash[d]
103
- new(
104
- ip: d.fetch("ip"),
105
- location: Location.from_dynamic!(d.fetch("location")),
106
- autonomous_system: AutonomousSystem.from_dynamic!(d.fetch("autonomous_system")),
107
- metadata: d,
108
- services: d.fetch("services", []).map { |x| Service.from_dynamic!(x) }
109
- )
182
+ class << self
183
+ #
184
+ # @param [Hash] d
185
+ #
186
+ # @return [Hit]
187
+ #
188
+ def from_dynamic!(d)
189
+ d = Types::Hash[d]
190
+ new(
191
+ ip: d.fetch("ip"),
192
+ location: Location.from_dynamic!(d.fetch("location")),
193
+ autonomous_system: AutonomousSystem.from_dynamic!(d.fetch("autonomous_system")),
194
+ metadata: d,
195
+ services: d.fetch("services", []).map { |x| Service.from_dynamic!(x) }
196
+ )
197
+ end
110
198
  end
111
199
  end
112
200
 
113
201
  class Links < Dry::Struct
114
- attribute :next, Types::String
115
- attribute :prev, Types::String
116
-
117
- def self.from_dynamic!(d)
118
- d = Types::Hash[d]
119
- new(
120
- next: d.fetch("next"),
121
- prev: d.fetch("prev")
122
- )
202
+ attribute :next, Types::String.optional
203
+ attribute :prev, Types::String.optional
204
+
205
+ #
206
+ # @return [String, nil]
207
+ #
208
+ def next
209
+ attributes[:next]
210
+ end
211
+
212
+ #
213
+ # @return [String, nil]
214
+ #
215
+ def prev
216
+ attributes[:prev]
217
+ end
218
+
219
+ class << self
220
+ #
221
+ # @param [Hash] d
222
+ #
223
+ # @return [Links]
224
+ #
225
+ def from_dynamic!(d)
226
+ d = Types::Hash[d]
227
+ new(
228
+ next: d["next"],
229
+ prev: d["prev"]
230
+ )
231
+ end
123
232
  end
124
233
  end
125
234
 
@@ -130,22 +239,55 @@ module Mihari
130
239
  attribute :links, Links
131
240
 
132
241
  #
133
- # @param [String] source
242
+ # @return [String]
243
+ #
244
+ def query
245
+ attributes[:query]
246
+ end
247
+
248
+ #
249
+ # @return [Integer]
250
+ #
251
+ def total
252
+ attributes[:total]
253
+ end
254
+
255
+ #
256
+ # @return [Array<Hit>]
257
+ #
258
+ def hits
259
+ attributes[:hits]
260
+ end
261
+
262
+ #
263
+ # @return [Links]
264
+ #
265
+ def links
266
+ attributes[:links]
267
+ end
268
+
134
269
  #
135
270
  # @return [Array<Mihari::Artifact>]
136
271
  #
137
- def to_artifacts(source = "Censys")
138
- hits.map { |hit| hit.to_artifact(source) }
272
+ def to_artifacts
273
+ hits.map(&:to_artifact)
139
274
  end
140
275
 
141
- def self.from_dynamic!(d)
142
- d = Types::Hash[d]
143
- new(
144
- query: d.fetch("query"),
145
- total: d.fetch("total"),
146
- hits: d.fetch("hits", []).map { |x| Hit.from_dynamic!(x) },
147
- links: Links.from_dynamic!(d.fetch("links"))
148
- )
276
+ class << self
277
+ #
278
+ # @param [Hash] d
279
+ #
280
+ # @return [Result]
281
+ #
282
+ def from_dynamic!(d)
283
+ d = Types::Hash[d]
284
+ new(
285
+ query: d.fetch("query"),
286
+ total: d.fetch("total"),
287
+ hits: d.fetch("hits", []).map { |x| Hit.from_dynamic!(x) },
288
+ links: Links.from_dynamic!(d.fetch("links"))
289
+ )
290
+ end
149
291
  end
150
292
  end
151
293
 
@@ -154,13 +296,41 @@ module Mihari
154
296
  attribute :status, Types::String
155
297
  attribute :result, Result
156
298
 
157
- def self.from_dynamic!(d)
158
- d = Types::Hash[d]
159
- new(
160
- code: d.fetch("code"),
161
- status: d.fetch("status"),
162
- result: Result.from_dynamic!(d.fetch("result"))
163
- )
299
+ #
300
+ # @return [Integer]
301
+ #
302
+ def code
303
+ attributes[:code]
304
+ end
305
+
306
+ #
307
+ # @return [String]
308
+ #
309
+ def status
310
+ attributes[:status]
311
+ end
312
+
313
+ #
314
+ # @return [Result]
315
+ #
316
+ def result
317
+ attributes[:result]
318
+ end
319
+
320
+ class << self
321
+ #
322
+ # @param [Hash] d
323
+ #
324
+ # @return [Response]
325
+ #
326
+ def from_dynamic!(d)
327
+ d = Types::Hash[d]
328
+ new(
329
+ code: d.fetch("code"),
330
+ status: d.fetch("status"),
331
+ result: Result.from_dynamic!(d.fetch("result"))
332
+ )
333
+ end
164
334
  end
165
335
  end
166
336
  end
@@ -9,31 +9,61 @@ module Mihari
9
9
  attribute :values, Types.Array(Types::Hash).optional
10
10
 
11
11
  #
12
- # @param [Class<Mihari::Analyzers::Base>, Class<Mihari::Emitters::Base>] klass
12
+ # @return [String]
13
13
  #
14
- # @return [Mihari::Structs::Config, nil] config
14
+ def name
15
+ attributes[:name]
16
+ end
17
+
18
+ #
19
+ # @return [String]
20
+ #
21
+ def type
22
+ attributes[:type]
23
+ end
24
+
25
+ #
26
+ # @return [Boolean]
15
27
  #
16
- def self.from_class(klass)
17
- return nil if klass == Mihari::Analyzers::Rule
28
+ def is_configured
29
+ attributes[:is_configured]
30
+ end
31
+
32
+ #
33
+ # @return [Array<Hash>]
34
+ #
35
+ def values
36
+ attributes[:values]
37
+ end
38
+
39
+ class << self
40
+ #
41
+ # @param [Class<Mihari::Analyzers::Base>, Class<Mihari::Emitters::Base>] klass
42
+ #
43
+ # @return [Mihari::Structs::Config, nil] config
44
+ #
45
+ def from_class(klass)
46
+ return nil if klass == Mihari::Analyzers::Rule
18
47
 
19
- name = klass.to_s.split("::").last.to_s
48
+ name = klass.to_s.split("::").last.to_s
20
49
 
21
- is_analyzer = klass.ancestors.include?(Mihari::Analyzers::Base)
22
- is_emitter = klass.ancestors.include?(Mihari::Emitters::Base)
23
- is_enricher = klass.ancestors.include?(Mihari::Enrichers::Base)
50
+ is_analyzer = klass.ancestors.include?(Mihari::Analyzers::Base)
51
+ is_emitter = klass.ancestors.include?(Mihari::Emitters::Base)
52
+ is_enricher = klass.ancestors.include?(Mihari::Enrichers::Base)
24
53
 
25
- type = "Analyzer"
26
- type = "Emitter" if is_emitter
27
- type = "Enricher" if is_enricher
54
+ type = "Analyzer"
55
+ type = "Emitter" if is_emitter
56
+ type = "Enricher" if is_enricher
28
57
 
29
- begin
30
- instance = is_analyzer ? klass.new("dummy") : klass.new
31
- is_configured = instance.configured?
32
- values = instance.configuration_values
58
+ begin
59
+ instance = is_analyzer ? klass.new("dummy") : klass.new(artifacts: [], rule: nil)
60
+ is_configured = instance.configured?
61
+ values = instance.configuration_values
33
62
 
34
- new(name: name, values: values, is_configured: is_configured, type: type)
35
- rescue ArgumentError => _e
36
- nil
63
+ new(name: name, values: values, is_configured: is_configured, type: type)
64
+ rescue ArgumentError => _e
65
+ nil
66
+ end
37
67
  end
38
68
  end
39
69
  end
@@ -9,32 +9,74 @@ module Mihari
9
9
  5 => "CNAME",
10
10
  16 => "TXT",
11
11
  28 => "AAAA"
12
- }
12
+ }.freeze
13
13
 
14
14
  class Answer < Dry::Struct
15
15
  attribute :name, Types::String
16
16
  attribute :data, Types::String
17
17
  attribute :resource_type, Types::String
18
18
 
19
- def self.from_dynamic!(d)
20
- d = Types::Hash[d]
21
- resource_type = INT_TYPE_TO_TYPE[d.fetch("type")]
22
- new(
23
- name: d.fetch("name"),
24
- data: d.fetch("data"),
25
- resource_type: resource_type
26
- )
19
+ #
20
+ # @return [String]
21
+ #
22
+ def name
23
+ attributes[:name]
24
+ end
25
+
26
+ #
27
+ # @return [String]
28
+ #
29
+ def data
30
+ attributes[:data]
31
+ end
32
+
33
+ #
34
+ # @return [String]
35
+ #
36
+ def resource_type
37
+ attributes[:resource_type]
38
+ end
39
+
40
+ class << self
41
+ #
42
+ # @param [Hash] d
43
+ #
44
+ # @return [Answer]
45
+ #
46
+ def from_dynamic!(d)
47
+ d = Types::Hash[d]
48
+ resource_type = INT_TYPE_TO_TYPE[d.fetch("type")]
49
+ new(
50
+ name: d.fetch("name"),
51
+ data: d.fetch("data"),
52
+ resource_type: resource_type
53
+ )
54
+ end
27
55
  end
28
56
  end
29
57
 
30
58
  class Response < Dry::Struct
31
59
  attribute :answers, Types.Array(Answer)
32
60
 
33
- def self.from_dynamic!(d)
34
- d = Types::Hash[d]
35
- new(
36
- answers: d.fetch("Answer", []).map { |x| Answer.from_dynamic!(x) }
37
- )
61
+ #
62
+ # @return [Array<Answer>]
63
+ #
64
+ def answers
65
+ attributes[:answers]
66
+ end
67
+
68
+ class << self
69
+ #
70
+ # @param [Hash] d
71
+ #
72
+ # @return [Response]
73
+ #
74
+ def from_dynamic!(d)
75
+ d = Types::Hash[d]
76
+ new(
77
+ answers: d.fetch("Answer", []).map { |x| Answer.from_dynamic!(x) }
78
+ )
79
+ end
38
80
  end
39
81
  end
40
82
  end