mihari 6.3.0 → 7.0.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (161) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +4 -10
  3. data/.rubocop.yml +2 -0
  4. data/Dockerfile +14 -0
  5. data/config.ru +5 -3
  6. data/docker-compose.yml +61 -0
  7. data/exe/mihari +2 -1
  8. data/lefthook.yml +8 -0
  9. data/lib/mihari/actor.rb +4 -4
  10. data/lib/mihari/analyzers/base.rb +16 -0
  11. data/lib/mihari/analyzers/binaryedge.rb +4 -2
  12. data/lib/mihari/analyzers/censys.rb +7 -5
  13. data/lib/mihari/analyzers/circl.rb +5 -3
  14. data/lib/mihari/analyzers/crtsh.rb +10 -2
  15. data/lib/mihari/analyzers/dnstwister.rb +1 -1
  16. data/lib/mihari/analyzers/feed.rb +12 -20
  17. data/lib/mihari/analyzers/fofa.rb +6 -8
  18. data/lib/mihari/analyzers/greynoise.rb +4 -2
  19. data/lib/mihari/analyzers/hunterhow.rb +4 -2
  20. data/lib/mihari/analyzers/onyphe.rb +4 -2
  21. data/lib/mihari/analyzers/otx.rb +5 -3
  22. data/lib/mihari/analyzers/passivetotal.rb +29 -12
  23. data/lib/mihari/analyzers/pulsedive.rb +5 -3
  24. data/lib/mihari/analyzers/securitytrails.rb +32 -8
  25. data/lib/mihari/analyzers/shodan.rb +4 -2
  26. data/lib/mihari/analyzers/urlscan.rb +4 -2
  27. data/lib/mihari/analyzers/virustotal.rb +5 -5
  28. data/lib/mihari/analyzers/virustotal_intelligence.rb +4 -2
  29. data/lib/mihari/analyzers/zoomeye.rb +4 -2
  30. data/lib/mihari/cli/{main.rb → application.rb} +17 -5
  31. data/lib/mihari/cli/artifact.rb +14 -0
  32. data/lib/mihari/cli/config.rb +14 -0
  33. data/lib/mihari/cli/rule.rb +1 -0
  34. data/lib/mihari/cli/tag.rb +14 -0
  35. data/lib/mihari/clients/base.rb +2 -2
  36. data/lib/mihari/clients/binaryedge.rb +2 -2
  37. data/lib/mihari/clients/crtsh.rb +3 -10
  38. data/lib/mihari/clients/fofa.rb +1 -1
  39. data/lib/mihari/clients/hunterhow.rb +1 -1
  40. data/lib/mihari/clients/mmdb.rb +28 -0
  41. data/lib/mihari/clients/passivetotal.rb +7 -20
  42. data/lib/mihari/clients/securitytrails.rb +19 -43
  43. data/lib/mihari/clients/shodan_internet_db.rb +28 -0
  44. data/lib/mihari/clients/the_hive.rb +7 -5
  45. data/lib/mihari/commands/alert.rb +53 -11
  46. data/lib/mihari/commands/artifact.rb +66 -0
  47. data/lib/mihari/commands/config.rb +23 -0
  48. data/lib/mihari/commands/database.rb +1 -1
  49. data/lib/mihari/commands/rule.rb +40 -27
  50. data/lib/mihari/commands/search.rb +10 -11
  51. data/lib/mihari/commands/sidekiq.rb +31 -0
  52. data/lib/mihari/commands/tag.rb +46 -0
  53. data/lib/mihari/commands/web.rb +6 -7
  54. data/lib/mihari/{mixins/autonomous_system.rb → concerns/autonomous_system_normalizable.rb} +5 -3
  55. data/lib/mihari/concerns/configurable.rb +72 -0
  56. data/lib/mihari/concerns/database_connectable.rb +16 -0
  57. data/lib/mihari/{mixins/unwrap_error.rb → concerns/error_unwrappable.rb} +5 -3
  58. data/lib/mihari/{mixins/falsepositive.rb → concerns/falsepositive_validatable.rb} +5 -3
  59. data/lib/mihari/{mixins/refang.rb → concerns/refangable.rb} +5 -3
  60. data/lib/mihari/{mixins → concerns}/retriable.rb +4 -2
  61. data/lib/mihari/config.rb +13 -12
  62. data/lib/mihari/database.rb +30 -42
  63. data/lib/mihari/emitters/database.rb +5 -6
  64. data/lib/mihari/emitters/misp.rb +4 -11
  65. data/lib/mihari/emitters/slack.rb +7 -5
  66. data/lib/mihari/emitters/the_hive.rb +8 -58
  67. data/lib/mihari/emitters/webhook.rb +6 -6
  68. data/lib/mihari/enrichers/google_public_dns.rb +1 -1
  69. data/lib/mihari/enrichers/mmdb.rb +28 -0
  70. data/lib/mihari/enrichers/shodan.rb +3 -5
  71. data/lib/mihari/enrichers/whois.rb +3 -3
  72. data/lib/mihari/entities/alert.rb +3 -10
  73. data/lib/mihari/entities/artifact.rb +6 -14
  74. data/lib/mihari/entities/config.rb +2 -2
  75. data/lib/mihari/entities/cpe.rb +1 -0
  76. data/lib/mihari/entities/dns.rb +1 -0
  77. data/lib/mihari/entities/geolocation.rb +1 -0
  78. data/lib/mihari/entities/ip_address.rb +1 -3
  79. data/lib/mihari/entities/messages.rb +17 -0
  80. data/lib/mihari/entities/pagination.rb +11 -0
  81. data/lib/mihari/entities/port.rb +1 -0
  82. data/lib/mihari/entities/reverse_dns.rb +1 -0
  83. data/lib/mihari/entities/rule.rb +2 -20
  84. data/lib/mihari/entities/tag.rb +2 -2
  85. data/lib/mihari/entities/whois.rb +1 -0
  86. data/lib/mihari/errors.rb +2 -4
  87. data/lib/mihari/http.rb +4 -0
  88. data/lib/mihari/models/alert.rb +21 -53
  89. data/lib/mihari/models/artifact.rb +61 -97
  90. data/lib/mihari/models/autonomous_system.rb +0 -24
  91. data/lib/mihari/models/concerns/searchable.rb +50 -0
  92. data/lib/mihari/models/cpe.rb +0 -23
  93. data/lib/mihari/models/dns.rb +0 -20
  94. data/lib/mihari/models/geolocation.rb +0 -24
  95. data/lib/mihari/models/port.rb +3 -10
  96. data/lib/mihari/models/reverse_dns.rb +0 -23
  97. data/lib/mihari/models/rule.rb +16 -57
  98. data/lib/mihari/models/tag.rb +17 -1
  99. data/lib/mihari/models/tagging.rb +1 -1
  100. data/lib/mihari/models/whois.rb +0 -17
  101. data/lib/mihari/rule.rb +35 -24
  102. data/lib/mihari/schemas/alert.rb +1 -0
  103. data/lib/mihari/schemas/analyzer.rb +3 -2
  104. data/lib/mihari/schemas/concerns/orrable.rb +24 -0
  105. data/lib/mihari/schemas/emitter.rb +1 -2
  106. data/lib/mihari/schemas/enricher.rb +3 -4
  107. data/lib/mihari/schemas/macros.rb +1 -1
  108. data/lib/mihari/schemas/options.rb +0 -2
  109. data/lib/mihari/schemas/rule.rb +1 -2
  110. data/lib/mihari/services/builders.rb +158 -0
  111. data/lib/mihari/services/creators.rb +22 -0
  112. data/lib/mihari/services/destroyers.rb +41 -0
  113. data/lib/mihari/services/enrichers.rb +25 -0
  114. data/lib/mihari/services/feed.rb +107 -0
  115. data/lib/mihari/services/getters.rb +58 -0
  116. data/lib/mihari/services/initializers.rb +22 -0
  117. data/lib/mihari/services/{alert_builder.rb → proxies.rb} +10 -40
  118. data/lib/mihari/services/searchers.rb +91 -0
  119. data/lib/mihari/sidekiq/application.rb +13 -0
  120. data/lib/mihari/sidekiq/jobs.rb +36 -0
  121. data/lib/mihari/structs/censys.rb +1 -1
  122. data/lib/mihari/structs/config.rb +10 -10
  123. data/lib/mihari/structs/filters.rb +12 -130
  124. data/lib/mihari/structs/google_public_dns.rb +1 -1
  125. data/lib/mihari/structs/greynoise.rb +1 -1
  126. data/lib/mihari/structs/mmdb.rb +115 -0
  127. data/lib/mihari/structs/onyphe.rb +1 -1
  128. data/lib/mihari/structs/shodan.rb +2 -2
  129. data/lib/mihari/version.rb +1 -1
  130. data/lib/mihari/web/{app.rb → application.rb} +28 -15
  131. data/lib/mihari/web/endpoints/alerts.rb +34 -73
  132. data/lib/mihari/web/endpoints/artifacts.rb +27 -111
  133. data/lib/mihari/web/endpoints/configs.rb +3 -5
  134. data/lib/mihari/web/endpoints/ip_addresses.rb +14 -15
  135. data/lib/mihari/web/endpoints/rules.rb +58 -130
  136. data/lib/mihari/web/endpoints/tags.rb +21 -17
  137. data/lib/mihari/web/middleware/capture_exceptions.rb +25 -0
  138. data/lib/mihari/web/middleware/{connection_adapter.rb → connection.rb} +4 -2
  139. data/lib/mihari/web/public/assets/index-cQUcyII5.js +1766 -0
  140. data/lib/mihari/web/public/assets/index-dVaNxqTC.css +1 -0
  141. data/lib/mihari/web/public/index.html +2 -2
  142. data/lib/mihari/web/public/redoc-static.html +385 -385
  143. data/lib/mihari.rb +56 -28
  144. data/mihari.gemspec +12 -4
  145. data/mkdocs.yml +5 -2
  146. data/requirements.txt +1 -1
  147. metadata +164 -34
  148. data/lib/mihari/commands/mixins.rb +0 -11
  149. data/lib/mihari/enrichers/ipinfo.rb +0 -52
  150. data/lib/mihari/entities/message.rb +0 -9
  151. data/lib/mihari/feed/parser.rb +0 -38
  152. data/lib/mihari/feed/reader.rb +0 -111
  153. data/lib/mihari/mixins/configurable.rb +0 -68
  154. data/lib/mihari/schemas/mixins.rb +0 -20
  155. data/lib/mihari/services/alert_runner.rb +0 -20
  156. data/lib/mihari/services/rule_builder.rb +0 -46
  157. data/lib/mihari/structs/ipinfo.rb +0 -53
  158. data/lib/mihari/web/endpoints/exports.rb +0 -0
  159. data/lib/mihari/web/middleware/error_notification_adapter.rb +0 -35
  160. data/lib/mihari/web/public/assets/index-81613_nX.js +0 -1763
  161. data/lib/mihari/web/public/assets/index-Wv6xUrTI.css +0 -1
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Mihari
4
+ module Commands
5
+ #
6
+ # Config sub-commands
7
+ #
8
+ module Config
9
+ class << self
10
+ def included(thor)
11
+ thor.class_eval do
12
+ desc "list", "List config"
13
+ def list
14
+ configs = Services::ConfigSearcher.call
15
+ data = configs.map { |config| Entities::Config.represent(config) }
16
+ puts JSON.pretty_generate(data.as_json)
17
+ end
18
+ end
19
+ end
20
+ end
21
+ end
22
+ end
23
+ end
@@ -9,7 +9,7 @@ module Mihari
9
9
  class << self
10
10
  def included(thor)
11
11
  thor.class_eval do
12
- include Mixins
12
+ include Concerns::DatabaseConnectable
13
13
 
14
14
  desc "migrate", "Migrate DB schemas"
15
15
  around :with_db_connection
@@ -1,7 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require "pathname"
4
-
5
3
  module Mihari
6
4
  module Commands
7
5
  #
@@ -9,9 +7,10 @@ module Mihari
9
7
  #
10
8
  module Rule
11
9
  class << self
10
+ # rubocop:disable Metrics/AbcSize
12
11
  def included(thor)
13
12
  thor.class_eval do
14
- include Dry::Monads[:try, :result]
13
+ include Concerns::DatabaseConnectable
15
14
 
16
15
  desc "validate [PATH]", "Validate a rule file"
17
16
  #
@@ -20,8 +19,7 @@ module Mihari
20
19
  # @param [String] path
21
20
  #
22
21
  def validate(path)
23
- res = Dry::Monads::Try[ValidationError] { Mihari::Rule.from_yaml File.read(path) }
24
- rule = res.value!
22
+ rule = Dry::Monads::Try[ValidationError] { Mihari::Rule.from_yaml File.read(path) }.value!
25
23
  puts rule.data.to_yaml
26
24
  end
27
25
 
@@ -31,38 +29,53 @@ module Mihari
31
29
  #
32
30
  # @param [String] path
33
31
  #
34
- #
35
32
  def init(path = "./rule.yml")
36
- warning = "#{path} exists. Do you want to overwrite it? (y/n)"
33
+ warning = "Do you want to overwrite it? (y/n)"
37
34
  return if Pathname(path).exist? && !(yes? warning)
38
35
 
39
- initialize_rule path
36
+ Services::RuleInitializer.call(path)
40
37
 
41
38
  puts "A new rule file has been initialized: #{path}."
42
39
  end
43
40
 
44
- no_commands do
45
- #
46
- # Create a new rule
47
- #
48
- # @param [String] path
49
- # @param [Dry::Files] files
50
- #
51
- # @return [nil]
52
- #
53
- def initialize_rule(path, files = Dry::Files.new)
54
- rule = Mihari::Rule.new(
55
- id: SecureRandom.uuid,
56
- title: "Title goes here",
57
- description: "Description goes here",
58
- created_on: Date.today,
59
- queries: []
60
- )
61
- files.write(path, rule.yaml)
62
- end
41
+ desc "list [QUERY]", "List/search rules"
42
+ around :with_db_connection
43
+ method_option :page, type: :numeric, default: 1
44
+ method_option :limit, type: :numeric, default: 10
45
+ #
46
+ # @param [String] q
47
+ #
48
+ def list(q = "")
49
+ filter = Structs::Filters::Search.new(q: q, page: options["page"], limit: options["limit"])
50
+ value = Services::RuleSearcher.result(filter).value!
51
+ data = Entities::RulesWithPagination.represent(
52
+ results: value.results,
53
+ total: value.total,
54
+ current_page: value.filter[:page].to_i,
55
+ page_size: value.filter[:limit].to_i
56
+ )
57
+ puts JSON.pretty_generate(data.as_json)
58
+ end
59
+
60
+ desc "get [ID]", "Get a rule"
61
+ around :with_db_connection
62
+ def get(id)
63
+ value = Services::RuleGetter.result(id).value!
64
+ data = Entities::Rule.represent(value)
65
+ puts JSON.pretty_generate(data.as_json)
66
+ end
67
+
68
+ desc "delete [ID]", "Delete a rule"
69
+ around :with_db_connection
70
+ #
71
+ # @param [String] id
72
+ #
73
+ def delete(id)
74
+ Services::RuleDestroyer.result(id).value!
63
75
  end
64
76
  end
65
77
  end
78
+ # rubocop:enable Metrics/AbcSize
66
79
  end
67
80
  end
68
81
  end
@@ -9,32 +9,31 @@ module Mihari
9
9
  class << self
10
10
  def included(thor)
11
11
  thor.class_eval do
12
- include Dry::Monads[:try, :result]
13
- include Mixins
12
+ include Concerns::DatabaseConnectable
14
13
 
15
- desc "search [PATH_OR_ID]", "Search by a rule (Outputs null if there is no new finding)"
14
+ desc "search [PATH_OR_ID]", "Search by a rule"
16
15
  around :with_db_connection
17
- method_option :force_overwrite, type: :boolean, aliases: "-f", desc: "Force overwriting a rule"
16
+ method_option :force_overwrite, type: :boolean, default: false, aliases: "-f",
17
+ desc: "Force overwriting a rule"
18
18
  #
19
19
  # Search by a rule
20
20
  #
21
21
  # @param [String] path_or_id
22
22
  #
23
23
  def search(path_or_id)
24
- result = Dry::Monads::Try[StandardError] do
24
+ force_overwrite = options["force_overwrite"] || false
25
+ message = "Are you sure you want to overwrite this rule? (y/n)"
26
+
27
+ # @type [Mihari::Models::Alert]
28
+ alert = Dry::Monads::Try[StandardError] do
25
29
  # @type [Mihari::Rule]
26
30
  rule = Services::RuleBuilder.call(path_or_id)
27
31
 
28
- force_overwrite = options["force_overwrite"] || false
29
- message = "There is a diff in the rule. Are you sure you want to overwrite the rule? (y/n)"
30
32
  exit 0 if rule.diff? && !force_overwrite && !yes?(message)
31
33
 
32
34
  rule.update_or_create
33
35
  rule.call
34
- end.to_result
35
-
36
- # @type [Mihari::Models::Alert]
37
- alert = result.value!
36
+ end.value!
38
37
  data = Entities::Alert.represent(alert)
39
38
  puts JSON.pretty_generate(data.as_json)
40
39
  end
@@ -0,0 +1,31 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Mihari
4
+ module Commands
5
+ #
6
+ # Sidekiq sub-commands
7
+ #
8
+ module Sidekiq
9
+ class << self
10
+ def included(thor)
11
+ thor.class_eval do
12
+ desc "sidekiq", "Start Sidekiq"
13
+ method_option :env, type: :string, default: "production", desc: "Environment"
14
+ method_option :concurrency, type: :numeric, default: 5, desc: "Sidekiq concurrency", aliases: "-c"
15
+ def sidekiq
16
+ require "sidekiq/cli"
17
+
18
+ ENV["APP_ENV"] ||= options["env"]
19
+ concurrency = options["concurrency"].to_s
20
+
21
+ cli = ::Sidekiq::CLI.instance
22
+ cli.parse ["-r", File.expand_path(File.join(__dir__, "..", "sidekiq", "application.rb")), "-c",
23
+ concurrency]
24
+ cli.run
25
+ end
26
+ end
27
+ end
28
+ end
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,46 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Mihari
4
+ module Commands
5
+ #
6
+ # Tag sub-commands
7
+ #
8
+ module Tag
9
+ class << self
10
+ def included(thor)
11
+ thor.class_eval do
12
+ include Concerns::DatabaseConnectable
13
+
14
+ desc "list", "List/search tags"
15
+ around :with_db_connection
16
+ method_option :page, type: :numeric, default: 1
17
+ method_option :limit, type: :numeric, default: 10
18
+ #
19
+ # @param [String] q
20
+ #
21
+ def list(q = "")
22
+ filter = Structs::Filters::Search.new(q: q, page: options["page"], limit: options["limit"])
23
+ value = Services::TagSearcher.result(filter).value!
24
+ data = Entities::TagsWithPagination.represent(
25
+ results: value.results,
26
+ total: value.total,
27
+ current_page: value.filter[:page].to_i,
28
+ page_size: value.filter[:limit].to_i
29
+ )
30
+ puts JSON.pretty_generate(data.as_json)
31
+ end
32
+
33
+ desc "delete [ID]", "Delete a tag"
34
+ around :with_db_connection
35
+ #
36
+ # @param [Integer] id
37
+ #
38
+ def delete(id)
39
+ Services::TagDestroyer.result(id).value!
40
+ end
41
+ end
42
+ end
43
+ end
44
+ end
45
+ end
46
+ end
@@ -9,20 +9,19 @@ module Mihari
9
9
  class << self
10
10
  def included(thor)
11
11
  thor.class_eval do
12
- desc "web", "Launch the web app"
12
+ desc "web", "Start the web app"
13
13
  method_option :port, type: :numeric, default: 9292, desc: "Hostname to listen on"
14
14
  method_option :host, type: :string, default: "localhost", desc: "Port to listen on"
15
15
  method_option :threads, type: :string, default: "0:5", desc: "min:max threads to use"
16
16
  method_option :verbose, type: :boolean, default: true, desc: "Report each request"
17
17
  method_option :worker_timeout, type: :numeric, default: 60, desc: "Worker timeout value (in seconds)"
18
- method_option :hide_config_values, type: :boolean, default: true,
19
- desc: "Whether to hide config values or not"
20
18
  method_option :open, type: :boolean, default: true, desc: "Whether to open the app in browser or not"
21
- method_option :rack_env, type: :string, default: "production", desc: "Rack environment"
19
+ method_option :env, type: :string, default: "production", desc: "Environment"
22
20
  def web
23
- Mihari.config.hide_config_values = options["hide_config_values"]
24
- # set rack env as production
25
- ENV["RACK_ENV"] ||= options["rack_env"]
21
+ require "mihari/web/application"
22
+
23
+ ENV["APP_ENV"] ||= options["env"]
24
+
26
25
  Mihari::Web::App.run!(
27
26
  port: options["port"],
28
27
  host: options["host"],
@@ -1,11 +1,13 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Mihari
4
- module Mixins
4
+ module Concerns
5
5
  #
6
- # Autonomous System mixin
6
+ # Autonomous System concern
7
7
  #
8
- module AutonomousSystem
8
+ module AutonomousSystemNormalizable
9
+ extend ActiveSupport::Concern
10
+
9
11
  #
10
12
  # Normalize ASN value
11
13
  #
@@ -0,0 +1,72 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Mihari
4
+ module Concerns
5
+ #
6
+ # Configurable concern
7
+ #
8
+ module Configurable
9
+ extend ActiveSupport::Concern
10
+
11
+ #
12
+ # Check whether there are configuration key-values or not
13
+ #
14
+ # @return [Boolean]
15
+ #
16
+ def configuration_keys?
17
+ return true if self.class.configuration_keys.empty?
18
+
19
+ self.class.configuration_keys.all? { |key| Mihari.config.send(key) }
20
+ end
21
+
22
+ #
23
+ # Check whether it is configured or not
24
+ #
25
+ # @return [Boolean]
26
+ #
27
+ def configured?
28
+ configuration_keys? || api_key?
29
+ end
30
+
31
+ class_methods do
32
+ #
33
+ # Configuration items
34
+ #
35
+ # @return [Array<Hash>] Configuration values as a list of hash. Returns nil if there is any keys.
36
+ #
37
+ def configuration_items
38
+ return nil if configuration_keys.empty?
39
+
40
+ configuration_keys.map do |key|
41
+ value = Mihari.config.send(key)
42
+ value = "REDACTED" if value && Mihari.config.hide_config_values
43
+ { key: key.upcase, value: value }
44
+ end
45
+ end
46
+
47
+ #
48
+ # Configuration keys
49
+ #
50
+ # @return [Array<String>] A list of configuration keys
51
+ #
52
+ def configuration_keys
53
+ []
54
+ end
55
+ end
56
+
57
+ private
58
+
59
+ #
60
+ # Check whether API key is set or not
61
+ #
62
+ # @return [Boolean]
63
+ #
64
+ def api_key?
65
+ value = method(:api_key).call
66
+ !value.nil?
67
+ rescue NameError
68
+ true
69
+ end
70
+ end
71
+ end
72
+ end
@@ -0,0 +1,16 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Mihari
4
+ module Concerns
5
+ #
6
+ # Database connectable concern
7
+ #
8
+ module DatabaseConnectable
9
+ extend ActiveSupport::Concern
10
+
11
+ def with_db_connection(&block)
12
+ Database.with_db_connection(&block)
13
+ end
14
+ end
15
+ end
16
+ end
@@ -1,11 +1,13 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Mihari
4
- module Mixins
4
+ module Concerns
5
5
  #
6
- # Unwrap error mixins
6
+ # Error unwrappable concern
7
7
  #
8
- module UnwrapError
8
+ module ErrorUnwrappable
9
+ extend ActiveSupport::Concern
10
+
9
11
  def unwrap_error(err)
10
12
  return err unless err.is_a?(Dry::Monads::UnwrapError)
11
13
 
@@ -1,11 +1,13 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Mihari
4
- module Mixins
4
+ module Concerns
5
5
  #
6
- # False positive mixins
6
+ # False positive validatable concern
7
7
  #
8
- module FalsePositive
8
+ module FalsePositiveValidatable
9
+ extend ActiveSupport::Concern
10
+
9
11
  prepend MemoWise
10
12
 
11
13
  #
@@ -1,11 +1,13 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Mihari
4
- module Mixins
4
+ module Concerns
5
5
  #
6
- # Refang mixin
6
+ # Refangable concern
7
7
  #
8
- module Refang
8
+ module Refangable
9
+ extend ActiveSupport::Concern
10
+
9
11
  #
10
12
  # Refang defanged indicator
11
13
  #
@@ -1,11 +1,13 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Mihari
4
- module Mixins
4
+ module Concerns
5
5
  #
6
- # Retriable mixin
6
+ # Retriable concern
7
7
  #
8
8
  module Retriable
9
+ extend ActiveSupport::Concern
10
+
9
11
  DEFAULT_ON = [
10
12
  Errno::ECONNRESET,
11
13
  Errno::ECONNABORTED,
data/lib/mihari/config.rb CHANGED
@@ -14,12 +14,11 @@ module Mihari
14
14
  censys_secret: nil,
15
15
  circl_passive_password: nil,
16
16
  circl_passive_username: nil,
17
- database_url: URI("sqlite3:///mihari.db"),
17
+ database_url: URI("sqlite3:mihari.db"),
18
18
  fofa_api_key: nil,
19
19
  fofa_email: nil,
20
20
  greynoise_api_key: nil,
21
21
  hunterhow_api_key: nil,
22
- ipinfo_api_key: nil,
23
22
  misp_api_key: nil,
24
23
  misp_url: nil,
25
24
  onyphe_api_key: nil,
@@ -32,11 +31,12 @@ module Mihari
32
31
  slack_channel: nil,
33
32
  slack_webhook_url: nil,
34
33
  thehive_api_key: nil,
35
- thehive_api_version: nil,
36
34
  thehive_url: nil,
37
35
  urlscan_api_key: nil,
38
36
  virustotal_api_key: nil,
39
37
  zoomeye_api_key: nil,
38
+ # sidekiq
39
+ sidekiq_redis_url: nil,
40
40
  # others
41
41
  hide_config_values: true,
42
42
  ignore_error: false,
@@ -63,7 +63,7 @@ module Mihari
63
63
  # @return [String, nil]
64
64
 
65
65
  # @!attribute [r] database_url
66
- # @return [URI, nil]
66
+ # @return [URI]
67
67
 
68
68
  # @!attribute [r] fofa_api_key
69
69
  # @return [String, nil]
@@ -77,9 +77,6 @@ module Mihari
77
77
  # @!attribute [r] hunterhow_api_key
78
78
  # @return [String, nil]
79
79
 
80
- # @!attribute [r] ipinfo_api_key
81
- # @return [String, nil]
82
-
83
80
  # @!attribute [r] misp_url
84
81
  # @return [String, nil]
85
82
 
@@ -119,9 +116,6 @@ module Mihari
119
116
  # @!attribute [r] thehive_api_key
120
117
  # @return [String, nil]
121
118
 
122
- # @!attribute [r] thehive_api_version
123
- # @return [String, nil]
124
-
125
119
  # @!attribute [r] urlscan_api_key
126
120
  # @return [String, nil]
127
121
 
@@ -158,11 +152,18 @@ module Mihari
158
152
  # @!attribute [r] ignore_error
159
153
  # @return [Boolean]
160
154
 
161
- # @!attribute [rw] hide_config_values
155
+ # @!attribute [r] hide_config_values
162
156
  # @return [Boolean]
163
157
 
158
+ # @!attribute [r] sidekiq_redis_url
159
+ # @return [URI, nil]
160
+
164
161
  def database_url=(val)
165
- super URI(val.to_s)
162
+ super(URI(val.to_s))
163
+ end
164
+
165
+ def sidekiq_redis_url=(val)
166
+ super(val.nil? ? val : URI(val.to_s))
166
167
  end
167
168
 
168
169
  #
@@ -3,18 +3,10 @@
3
3
  # Make possible to use upper case acronyms in class names
4
4
  ActiveSupport::Inflector.inflections(:en) { |inflect| inflect.acronym "CPE" }
5
5
 
6
- def env
7
- ENV["APP_ENV"] || ENV["RACK_ENV"]
8
- end
9
-
10
- def development_env?
11
- env == "development"
12
- end
13
-
14
6
  #
15
- # Mihari v5 DB schema
7
+ # Mihari v7 DB schema
16
8
  #
17
- class V5Schema < ActiveRecord::Migration[7.1]
9
+ class V7Schema < ActiveRecord::Migration[7.1]
18
10
  def change
19
11
  create_table :rules, id: :string, if_not_exists: true do |t|
20
12
  t.string :title, null: false
@@ -24,7 +16,7 @@ class V5Schema < ActiveRecord::Migration[7.1]
24
16
  end
25
17
 
26
18
  create_table :alerts, if_not_exists: true do |t|
27
- t.timestamps
19
+ t.datetime :created_at
28
20
 
29
21
  t.belongs_to :rule, foreign_key: true, type: :string, null: false
30
22
  end
@@ -33,8 +25,9 @@ class V5Schema < ActiveRecord::Migration[7.1]
33
25
  t.string :data, null: false
34
26
  t.string :data_type, null: false
35
27
  t.string :source
28
+ t.string :query
36
29
  t.json :metadata
37
- t.timestamps
30
+ t.datetime :created_at
38
31
 
39
32
  t.belongs_to :alert, foreign_key: true, null: false
40
33
  end
@@ -102,33 +95,20 @@ class V5Schema < ActiveRecord::Migration[7.1]
102
95
 
103
96
  create_table :taggings, if_not_exists: true do |t|
104
97
  t.integer :tag_id
105
- t.integer :alert_id
98
+ t.string :rule_id
106
99
  t.datetime :created_at
107
100
  end
108
101
 
109
102
  add_index :taggings, :tag_id, if_not_exists: true
110
- add_index :taggings, %i[tag_id alert_id], unique: true, if_not_exists: true
111
- end
112
- end
113
-
114
- class V61Schema < ActiveRecord::Migration[7.1]
115
- def change
116
- add_column :artifacts, :query, :string
103
+ add_index :taggings, %i[tag_id rule_id], unique: true, if_not_exists: true
117
104
  end
118
105
  end
119
106
 
120
- def adapter
121
- return "postgresql" if %w[postgresql postgres].include?(Mihari.config.database_url.scheme)
122
- return "mysql2" if Mihari.config.database_url.scheme == "mysql2"
123
-
124
- "sqlite3"
125
- end
126
-
127
107
  #
128
108
  # @return [Array<ActiveRecord::Migration>] schemas
129
109
  #
130
110
  def schemas
131
- [V5Schema, V61Schema]
111
+ [V7Schema]
132
112
  end
133
113
 
134
114
  module Mihari
@@ -150,37 +130,45 @@ module Mihari
150
130
  # Establish DB connection
151
131
  #
152
132
  def connect
153
- return if ActiveRecord::Base.connected?
154
-
155
- case adapter
156
- when "postgresql", "mysql2"
157
- ActiveRecord::Base.establish_connection Mihari.config.database_url.to_s
158
- else
159
- ActiveRecord::Base.establish_connection(
160
- adapter: adapter,
161
- database: Mihari.config.database_url.path[1..]
162
- )
163
- end
164
- ActiveRecord::Base.logger = Logger.new($stdout) if development_env?
133
+ return if connected?
134
+
135
+ ActiveRecord::Base.establish_connection Mihari.config.database_url.to_s
136
+ ActiveRecord::Base.logger = Logger.new($stdout) if Mihari.development?
137
+ end
138
+
139
+ #
140
+ # @return [Boolean]
141
+ #
142
+ def connected?
143
+ ActiveRecord::Base.connected?
165
144
  end
166
145
 
167
146
  #
168
147
  # Close DB connection(s)
169
148
  #
170
149
  def close
171
- return unless ActiveRecord::Base.connected?
150
+ return unless connected?
172
151
 
173
152
  ActiveRecord::Base.connection_handler.clear_active_connections!
174
153
  end
175
154
 
176
155
  def with_db_connection
177
- Mihari::Database.connect
156
+ Mihari::Database.connect unless connected?
178
157
  yield
179
158
  rescue ActiveRecord::StatementInvalid
180
159
  Mihari.logger.error("The DB migration is not yet complete. Please run 'mihari db migrate'.")
181
160
  ensure
182
161
  Mihari::Database.close
183
162
  end
163
+
164
+ private
165
+
166
+ def adapter
167
+ return "postgresql" if %w[postgresql postgres].include?(Mihari.config.database_url.scheme)
168
+ return "mysql2" if Mihari.config.database_url.scheme == "mysql2"
169
+
170
+ "sqlite3"
171
+ end
184
172
  end
185
173
  end
186
174
  end