mihari 5.4.2 → 5.4.4

Sign up to get free protection for your applications and to get access to all the features.
Files changed (90) hide show
  1. checksums.yaml +4 -4
  2. data/frontend/package-lock.json +2399 -1504
  3. data/frontend/package.json +22 -22
  4. data/lib/mihari/analyzers/base.rb +25 -14
  5. data/lib/mihari/analyzers/binaryedge.rb +2 -48
  6. data/lib/mihari/analyzers/censys.rb +4 -20
  7. data/lib/mihari/analyzers/circl.rb +3 -27
  8. data/lib/mihari/analyzers/crtsh.rb +2 -17
  9. data/lib/mihari/analyzers/dnstwister.rb +2 -4
  10. data/lib/mihari/analyzers/greynoise.rb +5 -4
  11. data/lib/mihari/analyzers/hunterhow.rb +8 -23
  12. data/lib/mihari/analyzers/onyphe.rb +5 -39
  13. data/lib/mihari/analyzers/otx.rb +3 -39
  14. data/lib/mihari/analyzers/passivetotal.rb +4 -42
  15. data/lib/mihari/analyzers/pulsedive.rb +1 -1
  16. data/lib/mihari/analyzers/rule.rb +18 -13
  17. data/lib/mihari/analyzers/securitytrails.rb +4 -42
  18. data/lib/mihari/analyzers/shodan.rb +7 -39
  19. data/lib/mihari/analyzers/urlscan.rb +3 -39
  20. data/lib/mihari/analyzers/virustotal.rb +1 -1
  21. data/lib/mihari/analyzers/virustotal_intelligence.rb +2 -25
  22. data/lib/mihari/analyzers/zoomeye.rb +18 -84
  23. data/lib/mihari/clients/base.rb +9 -1
  24. data/lib/mihari/clients/binaryedge.rb +26 -4
  25. data/lib/mihari/clients/censys.rb +32 -2
  26. data/lib/mihari/clients/circl.rb +28 -1
  27. data/lib/mihari/clients/crtsh.rb +7 -2
  28. data/lib/mihari/clients/dnstwister.rb +4 -2
  29. data/lib/mihari/clients/greynoise.rb +31 -4
  30. data/lib/mihari/clients/hunterhow.rb +41 -3
  31. data/lib/mihari/clients/onyphe.rb +25 -3
  32. data/lib/mihari/clients/otx.rb +40 -0
  33. data/lib/mihari/clients/passivetotal.rb +33 -15
  34. data/lib/mihari/clients/publsedive.rb +1 -1
  35. data/lib/mihari/clients/securitytrails.rb +44 -0
  36. data/lib/mihari/clients/shodan.rb +31 -3
  37. data/lib/mihari/clients/urlscan.rb +32 -6
  38. data/lib/mihari/clients/virustotal.rb +29 -4
  39. data/lib/mihari/clients/zoomeye.rb +53 -2
  40. data/lib/mihari/commands/alert.rb +42 -13
  41. data/lib/mihari/commands/rule.rb +11 -7
  42. data/lib/mihari/commands/search.rb +54 -22
  43. data/lib/mihari/commands/web.rb +1 -1
  44. data/lib/mihari/config.rb +6 -1
  45. data/lib/mihari/emitters/base.rb +9 -3
  46. data/lib/mihari/emitters/slack.rb +1 -1
  47. data/lib/mihari/enrichers/base.rb +13 -0
  48. data/lib/mihari/enrichers/google_public_dns.rb +16 -1
  49. data/lib/mihari/enrichers/ipinfo.rb +9 -13
  50. data/lib/mihari/enrichers/shodan.rb +1 -2
  51. data/lib/mihari/enrichers/whois.rb +2 -2
  52. data/lib/mihari/errors.rb +16 -10
  53. data/lib/mihari/feed/parser.rb +2 -2
  54. data/lib/mihari/models/artifact.rb +1 -1
  55. data/lib/mihari/models/autonomous_system.rb +11 -5
  56. data/lib/mihari/models/cpe.rb +10 -4
  57. data/lib/mihari/models/dns.rb +11 -16
  58. data/lib/mihari/models/geolocation.rb +11 -5
  59. data/lib/mihari/models/port.rb +10 -4
  60. data/lib/mihari/models/reverse_dns.rb +10 -4
  61. data/lib/mihari/models/whois.rb +4 -1
  62. data/lib/mihari/schemas/analyzer.rb +1 -0
  63. data/lib/mihari/services/alert_builder.rb +43 -0
  64. data/lib/mihari/services/alert_proxy.rb +7 -25
  65. data/lib/mihari/services/alert_runner.rb +9 -0
  66. data/lib/mihari/services/rule_builder.rb +47 -0
  67. data/lib/mihari/services/rule_proxy.rb +5 -61
  68. data/lib/mihari/services/rule_runner.rb +9 -4
  69. data/lib/mihari/structs/binaryedge.rb +89 -0
  70. data/lib/mihari/structs/censys.rb +11 -11
  71. data/lib/mihari/structs/greynoise.rb +17 -8
  72. data/lib/mihari/structs/onyphe.rb +7 -7
  73. data/lib/mihari/structs/shodan.rb +7 -6
  74. data/lib/mihari/structs/urlscan.rb +4 -6
  75. data/lib/mihari/structs/virustotal_intelligence.rb +4 -6
  76. data/lib/mihari/type_checker.rb +1 -1
  77. data/lib/mihari/version.rb +1 -1
  78. data/lib/mihari/web/endpoints/alerts.rb +33 -15
  79. data/lib/mihari/web/endpoints/artifacts.rb +53 -25
  80. data/lib/mihari/web/endpoints/configs.rb +2 -2
  81. data/lib/mihari/web/endpoints/ip_addresses.rb +3 -5
  82. data/lib/mihari/web/endpoints/rules.rb +97 -71
  83. data/lib/mihari/web/endpoints/tags.rb +15 -5
  84. data/lib/mihari/web/public/assets/index-ef33a6cd.js +1738 -0
  85. data/lib/mihari/web/public/index.html +1 -1
  86. data/lib/mihari/web/public/redoc-static.html +419 -382
  87. data/lib/mihari.rb +4 -0
  88. data/mihari.gemspec +10 -9
  89. metadata +38 -21
  90. data/lib/mihari/web/public/assets/index-4d7eda9f.js +0 -1738
@@ -21,9 +21,7 @@ module Mihari
21
21
  #
22
22
  def from_dynamic!(d)
23
23
  d = Types::Hash[d]
24
- new(
25
- url: d["url"]
26
- )
24
+ new(url: d["url"])
27
25
  end
28
26
  end
29
27
  end
@@ -81,7 +79,7 @@ module Mihari
81
79
  #
82
80
  # @return [Mihari::Artifact]
83
81
  #
84
- def to_artifact
82
+ def artifact
85
83
  Artifact.new(data: value, metadata: metadata)
86
84
  end
87
85
 
@@ -155,8 +153,8 @@ module Mihari
155
153
  #
156
154
  # @return [Array<Mihari::Artifact>]
157
155
  #
158
- def to_artifacts
159
- data.map(&:to_artifact)
156
+ def artifacts
157
+ data.map(&:artifact)
160
158
  end
161
159
 
162
160
  class << self
@@ -54,7 +54,7 @@ module Mihari
54
54
  return "ip" if ip?
55
55
  return "domain" if domain?
56
56
  return "url" if url?
57
- return "mail" if mail?
57
+ "mail" if mail?
58
58
  end
59
59
 
60
60
  # @return [String, nil]
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Mihari
4
- VERSION = "5.4.2"
4
+ VERSION = "5.4.4"
5
5
  end
@@ -54,18 +54,27 @@ module Mihari
54
54
  requires :id, type: Integer
55
55
  end
56
56
  delete "/:id" do
57
+ extend Dry::Monads[:result, :try]
58
+
57
59
  id = params["id"].to_i
58
60
 
59
- begin
61
+ result = Try do
60
62
  alert = Mihari::Alert.find(id)
61
- rescue ActiveRecord::RecordNotFound
62
- error!({ message: "ID:#{id} is not found" }, 404)
63
- end
63
+ alert.destroy
64
+ end.to_result
64
65
 
65
- alert.destroy
66
+ if result.success?
67
+ status 204
68
+ return present({ message: "" }, with: Entities::Message)
69
+ end
66
70
 
67
- status 204
68
- present({ message: "" }, with: Entities::Message)
71
+ failure = result.failure
72
+ case failure
73
+ when ActiveRecord::RecordNotFound
74
+ error!({ message: "ID:#{id} is not found" }, 404)
75
+ else
76
+ raise failure
77
+ end
69
78
  end
70
79
 
71
80
  desc "Create an alert", {
@@ -77,17 +86,26 @@ module Mihari
77
86
  requires :artifacts, type: Array, documentation: { type: String, is_array: true, param_type: "body" }
78
87
  end
79
88
  post "/" do
80
- proxy = Services::AlertProxy.new(params.to_snake_keys)
81
- runner = Services::AlertRunner.new(proxy)
89
+ extend Dry::Monads[:result, :try]
82
90
 
83
- begin
84
- alert = runner.run
85
- rescue ActiveRecord::RecordNotFound
86
- error!({ message: "Rule:#{params["ruleId"]} is not found" }, 404)
91
+ result = Try do
92
+ proxy = Services::AlertProxy.new(params.to_snake_keys)
93
+ runner = Services::AlertRunner.new(proxy)
94
+ runner.run
95
+ end.to_result
96
+
97
+ if result.success?
98
+ status 201
99
+ return present(result.value!, with: Entities::Alert)
87
100
  end
88
101
 
89
- status 201
90
- present alert, with: Entities::Alert
102
+ failure = result.failure
103
+ case failure
104
+ when ActiveRecord::RecordNotFound
105
+ error!({ message: "Rule:#{params["ruleId"]} is not found" }, 404)
106
+ else
107
+ raise failure
108
+ end
91
109
  end
92
110
  end
93
111
  end
@@ -13,9 +13,11 @@ module Mihari
13
13
  requires :id, type: Integer
14
14
  end
15
15
  get "/:id" do
16
+ extend Dry::Monads[:result, :try]
17
+
16
18
  id = params[:id].to_i
17
19
 
18
- begin
20
+ result = Try do
19
21
  artifact = Mihari::Artifact.includes(
20
22
  :autonomous_system,
21
23
  :geolocation,
@@ -23,18 +25,25 @@ module Mihari
23
25
  :dns_records,
24
26
  :reverse_dns_names
25
27
  ).find(id)
26
- rescue ActiveRecord::RecordNotFound
27
- error!({ message: "ID:#{id} is not found" }, 404)
28
- end
28
+ # TODO: improve queries
29
+ alert_ids = Mihari::Artifact.where(data: artifact.data).pluck(:alert_id)
30
+ tag_ids = Mihari::Tagging.where(alert_id: alert_ids).pluck(:tag_id)
31
+ tag_names = Mihari::Tag.where(id: tag_ids).distinct.pluck(:name)
29
32
 
30
- # TODO: improve queries
31
- alert_ids = Mihari::Artifact.where(data: artifact.data).pluck(:alert_id)
32
- tag_ids = Mihari::Tagging.where(alert_id: alert_ids).pluck(:tag_id)
33
- tag_names = Mihari::Tag.where(id: tag_ids).distinct.pluck(:name)
33
+ artifact.tags = tag_names
34
34
 
35
- artifact.tags = tag_names
35
+ artifact
36
+ end.to_result
36
37
 
37
- present artifact, with: Entities::Artifact
38
+ return present(result.value!, with: Entities::Artifact) if result.success?
39
+
40
+ failure = result.failure
41
+ case failure
42
+ when ActiveRecord::RecordNotFound
43
+ error!({ message: "ID:#{id} is not found" }, 404)
44
+ else
45
+ raise failure
46
+ end
38
47
  end
39
48
 
40
49
  desc "Enrich an artifact", {
@@ -46,9 +55,11 @@ module Mihari
46
55
  requires :id, type: Integer
47
56
  end
48
57
  get "/:id/enrich" do
58
+ extend Dry::Monads[:result, :try]
59
+
49
60
  id = params["id"].to_i
50
61
 
51
- begin
62
+ result = Try do
52
63
  artifact = Mihari::Artifact.includes(
53
64
  :autonomous_system,
54
65
  :geolocation,
@@ -58,15 +69,23 @@ module Mihari
58
69
  :cpes,
59
70
  :ports
60
71
  ).find(id)
61
- rescue ActiveRecord::RecordNotFound
62
- error!({ message: "ID:#{id} is not found" }, 404)
63
- end
64
72
 
65
- artifact.enrich_all
66
- artifact.save
73
+ artifact.enrich_all
74
+ artifact.save
75
+ end.to_result
67
76
 
68
- status 201
69
- present({ message: "" }, with: Entities::Message)
77
+ if result.success?
78
+ status 201
79
+ return present({ message: "" }, with: Entities::Message)
80
+ end
81
+
82
+ failure = result.failure
83
+ case failure
84
+ when ActiveRecord::RecordNotFound
85
+ error!({ message: "ID:#{id} is not found" }, 404)
86
+ else
87
+ raise failure
88
+ end
70
89
  end
71
90
 
72
91
  desc "Delete an artifact", {
@@ -78,18 +97,27 @@ module Mihari
78
97
  requires :id, type: Integer
79
98
  end
80
99
  delete "/:id" do
100
+ extend Dry::Monads[:result, :try]
101
+
81
102
  id = params["id"].to_i
82
103
 
83
- begin
104
+ result = Try do
84
105
  alert = Mihari::Artifact.find(id)
85
- rescue ActiveRecord::RecordNotFound
86
- error!({ message: "ID:#{id} is not found" }, 404)
87
- end
106
+ alert.destroy
107
+ end.to_result
88
108
 
89
- alert.destroy
109
+ if result.success?
110
+ status 204
111
+ return present({ message: "" }, with: Entities::Message)
112
+ end
90
113
 
91
- status 204
92
- present({ message: "" }, with: Entities::Message)
114
+ failure = result.failure
115
+ case failure
116
+ when ActiveRecord::RecordNotFound
117
+ error!({ message: "ID:#{id} is not found" }, 404)
118
+ else
119
+ raise failure
120
+ end
93
121
  end
94
122
  end
95
123
  end
@@ -10,9 +10,9 @@ module Mihari
10
10
  summary: "Get configs"
11
11
  }
12
12
  get "/" do
13
- configs = (Mihari.analyzers + Mihari.emitters + Mihari.enrichers).map do |klass|
13
+ configs = (Mihari.analyzers + Mihari.emitters + Mihari.enrichers).filter_map do |klass|
14
14
  Mihari::Structs::Config.from_class(klass)
15
- end.compact
15
+ end
16
16
 
17
17
  present(configs, with: Entities::Config)
18
18
  end
@@ -16,11 +16,9 @@ module Mihari
16
16
  ip = params[:ip].to_s
17
17
 
18
18
  data = Enrichers::IPInfo.query(ip)
19
- if data.nil?
20
- error!({ message: "IP:#{ip} is not found" }, 404)
21
- else
22
- present data, with: Entities::IPAddress
23
- end
19
+ error!({ message: "IP:#{ip} is not found" }, 404) if data.nil?
20
+
21
+ present data, with: Entities::IPAddress
24
22
  end
25
23
  end
26
24
  end
@@ -61,15 +61,23 @@ module Mihari
61
61
  requires :id, type: String
62
62
  end
63
63
  get "/:id" do
64
+ extend Dry::Monads[:result, :try]
65
+
64
66
  id = params["id"].to_s
65
67
 
66
- begin
67
- rule = Mihari::Rule.find(id)
68
- rescue ActiveRecord::RecordNotFound
68
+ result = Try do
69
+ Mihari::Rule.find(id)
70
+ end.to_result
71
+
72
+ return present(result.value!, with: Entities::Rule) if result.success?
73
+
74
+ failure = result.failure
75
+ case failure
76
+ when ActiveRecord::RecordNotFound
69
77
  error!({ message: "ID:#{id} is not found" }, 404)
78
+ else
79
+ raise failure
70
80
  end
71
-
72
- present rule, with: Entities::Rule
73
81
  end
74
82
 
75
83
  desc "Run a rule", {
@@ -80,19 +88,27 @@ module Mihari
80
88
  requires :id, type: String
81
89
  end
82
90
  get "/:id/run" do
91
+ extend Dry::Monads[:result, :try]
92
+
83
93
  id = params["id"].to_s
84
94
 
85
- begin
86
- rule = Mihari::Services::RuleProxy.from_model(Mihari::Rule.find(id))
87
- rescue ActiveRecord::RecordNotFound
88
- error!({ message: "ID:#{id} is not found" }, 404)
89
- end
95
+ result = Try do
96
+ Mihari::Services::RuleProxy.from_model(Mihari::Rule.find(id))
97
+ end.to_result
90
98
 
91
- analyzer = rule.analyzer
92
- analyzer.run
99
+ if result.success?
100
+ result.value!.analyzer.run
101
+ status 201
102
+ return present({ message: "ID:#{id} is ran successfully" }, with: Entities::Message)
103
+ end
93
104
 
94
- status 201
95
- present({ message: "ID:#{id} is ran successfully" }, with: Entities::Message)
105
+ failure = result.failure
106
+ case failure
107
+ when ActiveRecord::RecordNotFound
108
+ error!({ message: "ID:#{id} is not found" }, 404)
109
+ else
110
+ raise failure
111
+ end
96
112
  end
97
113
 
98
114
  desc "Create a rule", {
@@ -103,39 +119,38 @@ module Mihari
103
119
  requires :yaml, type: String, documentation: { param_type: "body" }
104
120
  end
105
121
  post "/" do
106
- yaml = params[:yaml]
122
+ extend Dry::Monads[:result, :try]
107
123
 
108
- begin
109
- rule = Services::RuleProxy.from_yaml(yaml)
110
- rescue YAMLSyntaxError => e
111
- error!({ message: e.message }, 400)
124
+ yaml = params[:yaml]
125
+ result = Try do
126
+ Services::RuleProxy.from_yaml(yaml)
127
+ end.to_result.bind do |rule|
128
+ Try do
129
+ found = Mihari::Rule.find_by_id(rule.id)
130
+ error!({ message: "ID:#{rule.id} is already registered" }, 400) unless found.nil?
131
+ rule
132
+ end.to_result
133
+ end.bind do |rule|
134
+ Try do
135
+ rule.model.save
136
+ rule
137
+ end.to_result
112
138
  end
113
139
 
114
- # check ID duplication
115
- begin
116
- Mihari::Rule.find(rule.id)
117
- error!({ message: "ID:#{rule.id} is already registered" }, 400)
118
- rescue ActiveRecord::RecordNotFound
119
- # do nothing
140
+ if result.success?
141
+ status 201
142
+ return present(result.value!.model, with: Entities::Rule)
120
143
  end
121
144
 
122
- begin
123
- rule.validate!
124
- rescue RuleValidationError
125
- error!({ message: "Data format is invalid", details: rule.errors.to_h }, 400) if rule.errors?
126
-
127
- # when NoMethodError occurs
128
- error!({ message: "Data format is invalid" }, 400)
145
+ failure = result.failure
146
+ case failure
147
+ when Psych::SyntaxError
148
+ error!({ message: failure.message }, 400)
149
+ when ValidationError
150
+ error!({ message: "Data format is invalid", details: failure.errors.to_h }, 400)
151
+ else
152
+ raise failure
129
153
  end
130
-
131
- begin
132
- rule.model.save
133
- rescue ActiveRecord::RecordNotUnique
134
- error!({ message: "ID:#{rule.id} is already registered" }, 400)
135
- end
136
-
137
- status 201
138
- present rule.model, with: Entities::Rule
139
154
  end
140
155
 
141
156
  desc "Update a rule", {
@@ -147,38 +162,40 @@ module Mihari
147
162
  requires :yaml, type: String, documentation: { param_type: "body" }
148
163
  end
149
164
  put "/" do
165
+ extend Dry::Monads[:result, :try]
166
+
150
167
  id = params[:id]
151
168
  yaml = params[:yaml]
152
169
 
153
- begin
170
+ result = Try do
154
171
  Mihari::Rule.find(id)
155
- rescue ActiveRecord::RecordNotFound
156
- error!({ message: "ID:#{id} is not found" }, 404)
172
+ end.to_result.bind do |_|
173
+ Try do
174
+ Services::RuleProxy.from_yaml(yaml)
175
+ end.to_result
176
+ end.bind do |rule|
177
+ Try do
178
+ rule.model.save
179
+ rule
180
+ end.to_result
157
181
  end
158
182
 
159
- begin
160
- rule = Services::RuleProxy.from_yaml(yaml)
161
- rescue YAMLSyntaxError => e
162
- error!({ message: e.message }, 400)
183
+ if result.success?
184
+ status 201
185
+ return present(result.value!.model, with: Entities::Rule)
163
186
  end
164
187
 
165
- begin
166
- rule.validate!
167
- rescue RuleValidationError
168
- error!({ message: "Data format is invalid", details: rule.errors.to_h }, 400) if rule.errors?
169
-
170
- # when NoMethodError occurs
171
- error!({ message: "Data format is invalid" }, 400)
172
- end
173
-
174
- begin
175
- rule.model.save
176
- rescue ActiveRecord::RecordNotUnique
177
- error!({ message: "ID:#{id} is already registered" }, 400)
188
+ failure = result.failure
189
+ case failure
190
+ when ActiveRecord::RecordNotFound
191
+ error!({ message: "ID:#{id} is not found" }, 404)
192
+ when Psych::SyntaxError
193
+ error!({ message: failure.message }, 400)
194
+ when ValidationError
195
+ error!({ message: "Data format is invalid", details: failure.errors.to_h }, 400)
196
+ else
197
+ raise failure
178
198
  end
179
-
180
- status 201
181
- present rule.model, with: Entities::Rule
182
199
  end
183
200
 
184
201
  desc "Delete a rule", {
@@ -190,18 +207,27 @@ module Mihari
190
207
  requires :id, type: String
191
208
  end
192
209
  delete "/:id" do
210
+ extend Dry::Monads[:result, :try]
211
+
193
212
  id = params["id"].to_s
194
213
 
195
- begin
214
+ result = Try do
196
215
  rule = Mihari::Rule.find(id)
197
- rescue ActiveRecord::RecordNotFound
198
- error!({ message: "ID:#{id} is not found" }, 404)
199
- end
216
+ rule.destroy
217
+ end.to_result
200
218
 
201
- rule.destroy
219
+ if result.success?
220
+ status 204
221
+ return present({ message: "ID:#{id} is deleted" }, with: Entities::Message)
222
+ end
202
223
 
203
- status 204
204
- present({ message: "ID:#{id} is deleted" }, with: Entities::Message)
224
+ failure = result.failure
225
+ case failure
226
+ when ActiveRecord::RecordNotFound
227
+ error!({ message: "ID:#{id} is not found" }, 404)
228
+ else
229
+ raise failure
230
+ end
205
231
  end
206
232
  end
207
233
  end
@@ -23,16 +23,26 @@ module Mihari
23
23
  requires :name, type: String
24
24
  end
25
25
  delete "/:name" do
26
+ extend Dry::Monads[:result, :try]
27
+
26
28
  name = params[:name].to_s
27
29
 
28
- begin
30
+ result = Try do
29
31
  Mihari::Tag.where(name: name).destroy_all
30
- rescue ActiveRecord::RecordNotFound
31
- error!({ message: "Name:#{name} is not found" }, 404)
32
+ end.to_result
33
+
34
+ if result.success?
35
+ status 204
36
+ return present({ message: "" }, with: Entities::Message)
32
37
  end
33
38
 
34
- status 204
35
- present({ message: "" }, with: Entities::Message)
39
+ failure = result.failure
40
+ case failure
41
+ when ActiveRecord::RecordNotFound
42
+ error!({ message: "Name:#{name} is not found" }, 404)
43
+ else
44
+ raise failure
45
+ end
36
46
  end
37
47
  end
38
48
  end