mihari 6.2.0 → 7.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (160) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +4 -10
  3. data/.rubocop.yml +2 -0
  4. data/Dockerfile +13 -0
  5. data/config.ru +5 -3
  6. data/docker-compose.yml +62 -0
  7. data/exe/mihari +2 -1
  8. data/lefthook.yml +8 -0
  9. data/lib/mihari/actor.rb +4 -4
  10. data/lib/mihari/analyzers/base.rb +16 -0
  11. data/lib/mihari/analyzers/binaryedge.rb +4 -2
  12. data/lib/mihari/analyzers/censys.rb +7 -5
  13. data/lib/mihari/analyzers/circl.rb +5 -3
  14. data/lib/mihari/analyzers/crtsh.rb +4 -1
  15. data/lib/mihari/analyzers/dnstwister.rb +1 -1
  16. data/lib/mihari/analyzers/feed.rb +12 -20
  17. data/lib/mihari/analyzers/fofa.rb +6 -8
  18. data/lib/mihari/analyzers/greynoise.rb +4 -2
  19. data/lib/mihari/analyzers/hunterhow.rb +4 -2
  20. data/lib/mihari/analyzers/onyphe.rb +4 -2
  21. data/lib/mihari/analyzers/otx.rb +5 -3
  22. data/lib/mihari/analyzers/passivetotal.rb +29 -12
  23. data/lib/mihari/analyzers/pulsedive.rb +5 -3
  24. data/lib/mihari/analyzers/securitytrails.rb +32 -8
  25. data/lib/mihari/analyzers/shodan.rb +4 -2
  26. data/lib/mihari/analyzers/urlscan.rb +4 -2
  27. data/lib/mihari/analyzers/virustotal.rb +5 -5
  28. data/lib/mihari/analyzers/virustotal_intelligence.rb +4 -2
  29. data/lib/mihari/analyzers/zoomeye.rb +4 -2
  30. data/lib/mihari/cli/{main.rb → application.rb} +17 -5
  31. data/lib/mihari/cli/artifact.rb +14 -0
  32. data/lib/mihari/cli/config.rb +14 -0
  33. data/lib/mihari/cli/rule.rb +1 -0
  34. data/lib/mihari/cli/tag.rb +14 -0
  35. data/lib/mihari/clients/base.rb +2 -2
  36. data/lib/mihari/clients/binaryedge.rb +2 -2
  37. data/lib/mihari/clients/crtsh.rb +2 -9
  38. data/lib/mihari/clients/fofa.rb +1 -1
  39. data/lib/mihari/clients/hunterhow.rb +1 -1
  40. data/lib/mihari/clients/mmdb.rb +28 -0
  41. data/lib/mihari/clients/passivetotal.rb +7 -20
  42. data/lib/mihari/clients/securitytrails.rb +19 -43
  43. data/lib/mihari/clients/shodan_internet_db.rb +28 -0
  44. data/lib/mihari/clients/the_hive.rb +7 -5
  45. data/lib/mihari/commands/alert.rb +53 -11
  46. data/lib/mihari/commands/artifact.rb +66 -0
  47. data/lib/mihari/commands/config.rb +23 -0
  48. data/lib/mihari/commands/database.rb +1 -1
  49. data/lib/mihari/commands/rule.rb +40 -27
  50. data/lib/mihari/commands/search.rb +10 -11
  51. data/lib/mihari/commands/sidekiq.rb +31 -0
  52. data/lib/mihari/commands/tag.rb +46 -0
  53. data/lib/mihari/commands/web.rb +6 -7
  54. data/lib/mihari/{mixins/autonomous_system.rb → concerns/autonomous_system_normalizable.rb} +5 -3
  55. data/lib/mihari/concerns/configurable.rb +72 -0
  56. data/lib/mihari/concerns/database_connectable.rb +16 -0
  57. data/lib/mihari/{mixins/unwrap_error.rb → concerns/error_unwrappable.rb} +5 -3
  58. data/lib/mihari/{mixins/falsepositive.rb → concerns/falsepositive_validatable.rb} +5 -3
  59. data/lib/mihari/{mixins/refang.rb → concerns/refangable.rb} +5 -3
  60. data/lib/mihari/{mixins → concerns}/retriable.rb +4 -2
  61. data/lib/mihari/config.rb +13 -12
  62. data/lib/mihari/database.rb +30 -42
  63. data/lib/mihari/emitters/database.rb +5 -6
  64. data/lib/mihari/emitters/misp.rb +4 -11
  65. data/lib/mihari/emitters/slack.rb +7 -5
  66. data/lib/mihari/emitters/the_hive.rb +8 -58
  67. data/lib/mihari/emitters/webhook.rb +6 -6
  68. data/lib/mihari/enrichers/google_public_dns.rb +1 -1
  69. data/lib/mihari/enrichers/mmdb.rb +28 -0
  70. data/lib/mihari/enrichers/shodan.rb +3 -5
  71. data/lib/mihari/enrichers/whois.rb +3 -3
  72. data/lib/mihari/entities/alert.rb +3 -10
  73. data/lib/mihari/entities/artifact.rb +9 -9
  74. data/lib/mihari/entities/config.rb +2 -2
  75. data/lib/mihari/entities/cpe.rb +1 -0
  76. data/lib/mihari/entities/dns.rb +1 -0
  77. data/lib/mihari/entities/geolocation.rb +1 -0
  78. data/lib/mihari/entities/ip_address.rb +1 -3
  79. data/lib/mihari/entities/messages.rb +17 -0
  80. data/lib/mihari/entities/pagination.rb +11 -0
  81. data/lib/mihari/entities/port.rb +1 -0
  82. data/lib/mihari/entities/reverse_dns.rb +1 -0
  83. data/lib/mihari/entities/rule.rb +2 -20
  84. data/lib/mihari/entities/tag.rb +2 -2
  85. data/lib/mihari/entities/whois.rb +1 -0
  86. data/lib/mihari/errors.rb +2 -4
  87. data/lib/mihari/http.rb +4 -0
  88. data/lib/mihari/models/alert.rb +21 -76
  89. data/lib/mihari/models/artifact.rb +51 -31
  90. data/lib/mihari/models/autonomous_system.rb +5 -13
  91. data/lib/mihari/models/concerns/searchable.rb +50 -0
  92. data/lib/mihari/models/cpe.rb +3 -10
  93. data/lib/mihari/models/dns.rb +2 -6
  94. data/lib/mihari/models/geolocation.rb +7 -12
  95. data/lib/mihari/models/port.rb +3 -10
  96. data/lib/mihari/models/reverse_dns.rb +3 -8
  97. data/lib/mihari/models/rule.rb +16 -60
  98. data/lib/mihari/models/tag.rb +17 -1
  99. data/lib/mihari/models/tagging.rb +1 -1
  100. data/lib/mihari/models/whois.rb +1 -4
  101. data/lib/mihari/rule.rb +35 -24
  102. data/lib/mihari/schemas/alert.rb +1 -0
  103. data/lib/mihari/schemas/analyzer.rb +2 -2
  104. data/lib/mihari/schemas/concerns/orrable.rb +24 -0
  105. data/lib/mihari/schemas/emitter.rb +1 -2
  106. data/lib/mihari/schemas/enricher.rb +3 -4
  107. data/lib/mihari/schemas/macros.rb +1 -1
  108. data/lib/mihari/schemas/options.rb +0 -2
  109. data/lib/mihari/schemas/rule.rb +1 -2
  110. data/lib/mihari/services/{rule_builder.rb → builders.rb} +1 -6
  111. data/lib/mihari/services/creators.rb +22 -0
  112. data/lib/mihari/services/destroyers.rb +41 -0
  113. data/lib/mihari/services/enrichers.rb +25 -0
  114. data/lib/mihari/services/feed.rb +107 -0
  115. data/lib/mihari/services/getters.rb +58 -0
  116. data/lib/mihari/services/initializers.rb +22 -0
  117. data/lib/mihari/services/{alert_builder.rb → proxies.rb} +10 -40
  118. data/lib/mihari/services/searchers.rb +91 -0
  119. data/lib/mihari/sidekiq/application.rb +13 -0
  120. data/lib/mihari/sidekiq/jobs.rb +36 -0
  121. data/lib/mihari/structs/censys.rb +1 -1
  122. data/lib/mihari/structs/config.rb +10 -10
  123. data/lib/mihari/structs/filters.rb +12 -86
  124. data/lib/mihari/structs/google_public_dns.rb +1 -1
  125. data/lib/mihari/structs/greynoise.rb +1 -1
  126. data/lib/mihari/structs/mmdb.rb +115 -0
  127. data/lib/mihari/structs/onyphe.rb +1 -1
  128. data/lib/mihari/structs/shodan.rb +2 -2
  129. data/lib/mihari/version.rb +1 -1
  130. data/lib/mihari/web/{app.rb → application.rb} +28 -15
  131. data/lib/mihari/web/endpoints/alerts.rb +34 -81
  132. data/lib/mihari/web/endpoints/artifacts.rb +43 -63
  133. data/lib/mihari/web/endpoints/configs.rb +3 -5
  134. data/lib/mihari/web/endpoints/ip_addresses.rb +14 -15
  135. data/lib/mihari/web/endpoints/rules.rb +58 -137
  136. data/lib/mihari/web/endpoints/tags.rb +21 -17
  137. data/lib/mihari/web/middleware/capture_exceptions.rb +25 -0
  138. data/lib/mihari/web/middleware/{connection_adapter.rb → connection.rb} +4 -2
  139. data/lib/mihari/web/public/assets/index-cQUcyII5.js +1766 -0
  140. data/lib/mihari/web/public/assets/index-dVaNxqTC.css +1 -0
  141. data/lib/mihari/web/public/index.html +2 -3
  142. data/lib/mihari/web/public/redoc-static.html +385 -381
  143. data/lib/mihari.rb +56 -28
  144. data/mihari.gemspec +16 -8
  145. data/mkdocs.yml +5 -2
  146. data/requirements.txt +1 -1
  147. metadata +173 -42
  148. data/lib/mihari/commands/mixins.rb +0 -11
  149. data/lib/mihari/enrichers/ipinfo.rb +0 -52
  150. data/lib/mihari/entities/message.rb +0 -9
  151. data/lib/mihari/feed/parser.rb +0 -38
  152. data/lib/mihari/feed/reader.rb +0 -111
  153. data/lib/mihari/mixins/configurable.rb +0 -68
  154. data/lib/mihari/schemas/mixins.rb +0 -20
  155. data/lib/mihari/services/alert_runner.rb +0 -20
  156. data/lib/mihari/structs/ipinfo.rb +0 -53
  157. data/lib/mihari/web/middleware/error_notification_adapter.rb +0 -35
  158. data/lib/mihari/web/public/assets/index-1d77cd61.js +0 -1756
  159. data/lib/mihari/web/public/assets/index-4c8509ee.css +0 -1
  160. /data/lib/mihari/web/public/assets/{mode-yaml-24faa242.js → mode-yaml-BC4MIiYj.js} +0 -0
data/lib/mihari.rb CHANGED
@@ -1,21 +1,22 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  # standard libs
4
+ require "date"
5
+ require "erb"
4
6
  require "ipaddr"
5
7
  require "json"
8
+ require "pathname"
6
9
  require "resolv"
10
+ require "securerandom"
7
11
  require "yaml"
8
12
 
9
13
  # Active Support & Active Record
10
14
  require "active_support"
11
-
12
- require "active_support/core_ext/hash"
13
- require "active_support/core_ext/integer/time"
14
- require "active_support/core_ext/numeric/time"
15
- require "active_support/core_ext/object/deep_dup"
16
-
17
15
  require "active_record"
18
16
 
17
+ # Search Cop
18
+ require "search_cop"
19
+
19
20
  # dry-rb
20
21
  require "dry/files"
21
22
  require "dry/monads"
@@ -49,13 +50,14 @@ require "mihari/errors"
49
50
 
50
51
  require "mihari/config"
51
52
 
52
- # Mixins
53
- require "mihari/mixins/autonomous_system"
54
- require "mihari/mixins/configurable"
55
- require "mihari/mixins/falsepositive"
56
- require "mihari/mixins/refang"
57
- require "mihari/mixins/retriable"
58
- require "mihari/mixins/unwrap_error"
53
+ # Concerns
54
+ require "mihari/concerns/autonomous_system_normalizable"
55
+ require "mihari/concerns/configurable"
56
+ require "mihari/concerns/database_connectable"
57
+ require "mihari/concerns/error_unwrappable"
58
+ require "mihari/concerns/falsepositive_validatable"
59
+ require "mihari/concerns/refangable"
60
+ require "mihari/concerns/retriable"
59
61
 
60
62
  #
61
63
  # Mihari module
@@ -129,6 +131,27 @@ module Mihari
129
131
  end
130
132
  memo_wise :logger
131
133
 
134
+ #
135
+ # @return [String]
136
+ #
137
+ def env
138
+ ENV["APP_ENV"] || ENV["RACK_ENV"]
139
+ end
140
+
141
+ #
142
+ # @return [Boolean]
143
+ #
144
+ def development?
145
+ env == "development"
146
+ end
147
+
148
+ #
149
+ # @return [Boolean]
150
+ #
151
+ def sidekiq?
152
+ !Mihari.config.sidekiq_redis_url.nil?
153
+ end
154
+
132
155
  def initialize_sentry
133
156
  return if Mihari.config.sentry_dsn.nil?
134
157
  return if Sentry.initialized?
@@ -136,6 +159,7 @@ module Mihari
136
159
  Sentry.init do |config|
137
160
  config.dsn = Mihari.config.sentry_dsn
138
161
  config.traces_sample_rate = Mihari.config.sentry_trace_sample_rate
162
+ config.breadcrumbs_logger = %i[sentry_logger http_logger]
139
163
  end
140
164
  end
141
165
  end
@@ -154,11 +178,13 @@ require "mihari/rule"
154
178
  # Enrichers
155
179
  require "mihari/enrichers/base"
156
180
  require "mihari/enrichers/google_public_dns"
157
- require "mihari/enrichers/ipinfo"
181
+ require "mihari/enrichers/mmdb"
158
182
  require "mihari/enrichers/shodan"
159
183
  require "mihari/enrichers/whois"
160
184
 
161
185
  # Models
186
+ require "mihari/models/concerns/searchable"
187
+
162
188
  require "mihari/models/alert"
163
189
  require "mihari/models/artifact"
164
190
  require "mihari/models/autonomous_system"
@@ -194,11 +220,13 @@ require "mihari/clients/google_public_dns"
194
220
  require "mihari/clients/greynoise"
195
221
  require "mihari/clients/hunterhow"
196
222
  require "mihari/clients/misp"
223
+ require "mihari/clients/mmdb"
197
224
  require "mihari/clients/onyphe"
198
225
  require "mihari/clients/otx"
199
226
  require "mihari/clients/passivetotal"
200
227
  require "mihari/clients/publsedive"
201
228
  require "mihari/clients/securitytrails"
229
+ require "mihari/clients/shodan_internet_db"
202
230
  require "mihari/clients/shodan"
203
231
  require "mihari/clients/the_hive"
204
232
  require "mihari/clients/urlscan"
@@ -242,8 +270,8 @@ require "mihari/structs/filters"
242
270
  require "mihari/structs/fofa"
243
271
  require "mihari/structs/google_public_dns"
244
272
  require "mihari/structs/greynoise"
245
- require "mihari/structs/ipinfo"
246
273
  require "mihari/structs/hunterhow"
274
+ require "mihari/structs/mmdb"
247
275
  require "mihari/structs/onyphe"
248
276
  require "mihari/structs/shodan"
249
277
  require "mihari/structs/urlscan"
@@ -251,7 +279,7 @@ require "mihari/structs/virustotal_intelligence"
251
279
 
252
280
  # Schemas
253
281
  require "mihari/schemas/macros"
254
- require "mihari/schemas/mixins"
282
+ require "mihari/schemas/concerns/orrable"
255
283
 
256
284
  require "mihari/schemas/options"
257
285
 
@@ -260,13 +288,18 @@ require "mihari/schemas/analyzer"
260
288
  require "mihari/schemas/rule"
261
289
 
262
290
  # Services
263
- require "mihari/services/rule_builder"
264
-
265
- require "mihari/services/alert_builder"
266
- require "mihari/services/alert_runner"
291
+ require "mihari/services/builders"
292
+ require "mihari/services/creators"
293
+ require "mihari/services/destroyers"
294
+ require "mihari/services/enrichers"
295
+ require "mihari/services/feed"
296
+ require "mihari/services/getters"
297
+ require "mihari/services/initializers"
298
+ require "mihari/services/proxies"
299
+ require "mihari/services/searchers"
267
300
 
268
301
  # Entities
269
- require "mihari/entities/message"
302
+ require "mihari/entities/pagination"
270
303
 
271
304
  require "mihari/entities/autonomous_system"
272
305
  require "mihari/entities/config"
@@ -274,6 +307,7 @@ require "mihari/entities/cpe"
274
307
  require "mihari/entities/dns"
275
308
  require "mihari/entities/geolocation"
276
309
  require "mihari/entities/ip_address"
310
+ require "mihari/entities/messages"
277
311
  require "mihari/entities/port"
278
312
  require "mihari/entities/reverse_dns"
279
313
  require "mihari/entities/tag"
@@ -285,11 +319,5 @@ require "mihari/entities/alert"
285
319
 
286
320
  require "mihari/entities/rule"
287
321
 
288
- # Web app
289
- require "mihari/web/app"
290
-
291
- # CLIs
292
- require "mihari/cli/main"
293
-
294
- # initialize Sentry (if it's possible)
322
+ # Initialize Sentry (if it's possible)
295
323
  Mihari.initialize_sentry
data/mihari.gemspec CHANGED
@@ -38,8 +38,10 @@ Gem::Specification.new do |spec|
38
38
 
39
39
  spec.add_development_dependency "better_errors", "~> 2.10"
40
40
  spec.add_development_dependency "binding_of_caller", "~> 1.0"
41
- spec.add_development_dependency "bundler", "~> 2.4"
41
+ spec.add_development_dependency "bundler", "~> 2.5"
42
+ spec.add_development_dependency "capybara", "~> 3.39"
42
43
  spec.add_development_dependency "coveralls_reborn", "~> 0.28"
44
+ spec.add_development_dependency "factory_bot", "~> 6.4"
43
45
  spec.add_development_dependency "fakefs", "~> 2.5"
44
46
  spec.add_development_dependency "faker", "~> 3.2"
45
47
  spec.add_development_dependency "fuubar", "~> 2.5"
@@ -52,9 +54,11 @@ Gem::Specification.new do |spec|
52
54
  spec.add_development_dependency "rspec", "~> 3.12"
53
55
  spec.add_development_dependency "rspec-parameterized", "~> 1.0"
54
56
  spec.add_development_dependency "rubocop-rspec", "~> 2.25"
55
- spec.add_development_dependency "rubocop-yard", "~> 0.8"
57
+ spec.add_development_dependency "rubocop-yard", "~> 0.9"
58
+ spec.add_development_dependency "rubocop-rake", "~> 0.6"
59
+ spec.add_development_dependency "rubocop-factory_bot", "~> 2.24"
56
60
  spec.add_development_dependency "simplecov-lcov", "~> 0.8"
57
- spec.add_development_dependency "standard", "~> 1.32"
61
+ spec.add_development_dependency "standard", "~> 1.33"
58
62
  spec.add_development_dependency "test-prof", "~> 1.3"
59
63
  spec.add_development_dependency "timecop", "~> 0.9"
60
64
  spec.add_development_dependency "vcr", "~> 6.2"
@@ -63,11 +67,11 @@ Gem::Specification.new do |spec|
63
67
  unless ci_env?
64
68
  spec.add_development_dependency "lefthook", "~> 1.5"
65
69
  spec.add_development_dependency "ruby-lsp-rspec", "~> 0.1"
66
- spec.add_development_dependency "solargraph", "~> 0.49"
70
+ spec.add_development_dependency "solargraph", "~> 0.50"
67
71
  end
68
72
 
69
73
  spec.add_dependency "activerecord", "7.1.2"
70
- spec.add_dependency "addressable", "2.8.5"
74
+ spec.add_dependency "addressable", "~> 2.8"
71
75
  spec.add_dependency "anyway_config", "2.5.4"
72
76
  spec.add_dependency "awrence", "2.0.1"
73
77
  spec.add_dependency "dotenv", "2.8.1"
@@ -87,17 +91,21 @@ Gem::Specification.new do |spec|
87
91
  spec.add_dependency "launchy", "2.5.2"
88
92
  spec.add_dependency "memo_wise", "1.8.0"
89
93
  spec.add_dependency "normalize_country", "0.3.2"
90
- spec.add_dependency "parallel", "1.23.0"
94
+ spec.add_dependency "parallel", "1.24.0"
91
95
  spec.add_dependency "plissken", "2.0.1"
92
96
  spec.add_dependency "public_suffix", "5.0.4"
93
97
  spec.add_dependency "puma", "6.4.0"
94
98
  spec.add_dependency "rack", "3.0.8"
95
99
  spec.add_dependency "rack-cors", "2.0.1"
100
+ spec.add_dependency "rack-session", "2.0.0"
96
101
  spec.add_dependency "rackup", "2.1.0"
102
+ spec.add_dependency "search_cop", "1.2.3"
97
103
  spec.add_dependency "semantic_logger", "4.15.0"
98
- spec.add_dependency "sentry-ruby", "~> 5.14"
104
+ spec.add_dependency "sentry-ruby", "~> 5.15"
105
+ spec.add_dependency "sentry-sidekiq", "~> 5.15"
106
+ spec.add_dependency "sidekiq", "7.2.0"
99
107
  spec.add_dependency "slack-notifier", "2.4.0"
100
- spec.add_dependency "sqlite3", "~> 1.6"
108
+ spec.add_dependency "sqlite3", "~> 1.7"
101
109
  spec.add_dependency "thor", "1.3.0"
102
110
  spec.add_dependency "thor-hollaback", "0.2.1"
103
111
  spec.add_dependency "uuidtools", "2.2.0"
data/mkdocs.yml CHANGED
@@ -11,11 +11,13 @@ plugins:
11
11
 
12
12
  nav:
13
13
  - Requirements: requirements.md
14
+ - Overview: overview.md
14
15
  - Installation: installation.md
15
- - How to Write a Rule: rule.md
16
- - Usage: usage.md
16
+ - Rule: rule.md
17
17
  - Configuration: configuration.md
18
+ - Usage: usage.md
18
19
  - Tips:
20
+ - Sidekiq: tips/sidekiq.md
19
21
  - Docker: tips/docker.md
20
22
  - GitHub Actions: tips/github_actions.md
21
23
  - Superset: tips/superset.md
@@ -27,6 +29,7 @@ nav:
27
29
  - Emitters: emitters/index.md
28
30
  - Database: database.md
29
31
  - Tags: tags.md
32
+ - Development: dev.md
30
33
 
31
34
  markdown_extensions:
32
35
  - toc:
data/requirements.txt CHANGED
@@ -1,2 +1,2 @@
1
1
  mkdocs==1.5.3
2
- mkdocs-material==9.4.14
2
+ mkdocs-material==9.5.3