mihari 2.4.0 → 3.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (113) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +7 -0
  3. data/.overcommit.yml +12 -0
  4. data/README.md +1 -9
  5. data/build_frontend.sh +5 -0
  6. data/docker/Dockerfile +1 -1
  7. data/exe/mihari +1 -1
  8. data/lib/mihari.rb +89 -15
  9. data/lib/mihari/analyzers/base.rb +49 -8
  10. data/lib/mihari/analyzers/basic.rb +1 -2
  11. data/lib/mihari/analyzers/binaryedge.rb +7 -13
  12. data/lib/mihari/analyzers/censys.rb +26 -63
  13. data/lib/mihari/analyzers/circl.rb +20 -17
  14. data/lib/mihari/analyzers/crtsh.rb +6 -13
  15. data/lib/mihari/analyzers/dnpedia.rb +6 -12
  16. data/lib/mihari/analyzers/dnstwister.rb +13 -10
  17. data/lib/mihari/analyzers/onyphe.rb +6 -12
  18. data/lib/mihari/analyzers/otx.rb +22 -19
  19. data/lib/mihari/analyzers/passivetotal.rb +22 -21
  20. data/lib/mihari/analyzers/pulsedive.rb +16 -13
  21. data/lib/mihari/analyzers/rule.rb +97 -0
  22. data/lib/mihari/analyzers/securitytrails.rb +22 -19
  23. data/lib/mihari/analyzers/shodan.rb +7 -13
  24. data/lib/mihari/analyzers/spyse.rb +12 -19
  25. data/lib/mihari/analyzers/urlscan.rb +22 -27
  26. data/lib/mihari/analyzers/virustotal.rb +25 -22
  27. data/lib/mihari/analyzers/zoomeye.rb +14 -20
  28. data/lib/mihari/cli/analyzer.rb +44 -0
  29. data/lib/mihari/cli/base.rb +27 -0
  30. data/lib/mihari/cli/init.rb +13 -0
  31. data/lib/mihari/cli/main.rb +30 -0
  32. data/lib/mihari/cli/mixins/utils.rb +88 -0
  33. data/lib/mihari/cli/validator.rb +11 -0
  34. data/lib/mihari/commands/binaryedge.rb +1 -1
  35. data/lib/mihari/commands/censys.rb +1 -1
  36. data/lib/mihari/commands/circl.rb +2 -2
  37. data/lib/mihari/commands/crtsh.rb +1 -1
  38. data/lib/mihari/commands/dnpedia.rb +1 -1
  39. data/lib/mihari/commands/dnstwister.rb +2 -2
  40. data/lib/mihari/commands/init.rb +46 -0
  41. data/lib/mihari/commands/json.rb +1 -1
  42. data/lib/mihari/commands/onyphe.rb +1 -1
  43. data/lib/mihari/commands/otx.rb +2 -2
  44. data/lib/mihari/commands/passivetotal.rb +2 -2
  45. data/lib/mihari/commands/pulsedive.rb +2 -2
  46. data/lib/mihari/commands/search.rb +77 -0
  47. data/lib/mihari/commands/securitytrails.rb +2 -2
  48. data/lib/mihari/commands/shodan.rb +1 -1
  49. data/lib/mihari/commands/spyse.rb +1 -1
  50. data/lib/mihari/commands/urlscan.rb +2 -2
  51. data/lib/mihari/commands/validator.rb +38 -0
  52. data/lib/mihari/commands/virustotal.rb +2 -2
  53. data/lib/mihari/commands/zoomeye.rb +1 -1
  54. data/lib/mihari/constraints.rb +5 -0
  55. data/lib/mihari/database.rb +13 -2
  56. data/lib/mihari/emitters/base.rb +2 -2
  57. data/lib/mihari/emitters/database.rb +1 -1
  58. data/lib/mihari/emitters/misp.rb +3 -1
  59. data/lib/mihari/emitters/slack.rb +6 -7
  60. data/lib/mihari/emitters/the_hive.rb +1 -1
  61. data/lib/mihari/emitters/webhook.rb +2 -9
  62. data/lib/mihari/mixins/configurable.rb +38 -0
  63. data/lib/mihari/mixins/configuration.rb +90 -0
  64. data/lib/mihari/mixins/hash.rb +20 -0
  65. data/lib/mihari/mixins/refang.rb +21 -0
  66. data/lib/mihari/mixins/retriable.rb +27 -0
  67. data/lib/mihari/mixins/rule.rb +79 -0
  68. data/lib/mihari/models/alert.rb +28 -1
  69. data/lib/mihari/models/artifact.rb +11 -1
  70. data/lib/mihari/notifiers/base.rb +9 -1
  71. data/lib/mihari/notifiers/exception_notifier.rb +50 -0
  72. data/lib/mihari/notifiers/slack.rb +29 -0
  73. data/lib/mihari/schemas/analyzer.rb +25 -0
  74. data/lib/mihari/schemas/configuration.rb +42 -0
  75. data/lib/mihari/schemas/macros.rb +17 -0
  76. data/lib/mihari/schemas/rule.rb +72 -0
  77. data/lib/mihari/serializers/artifact.rb +1 -1
  78. data/lib/mihari/status.rb +14 -0
  79. data/lib/mihari/templates/rule.yml.erb +19 -0
  80. data/lib/mihari/type_checker.rb +8 -3
  81. data/lib/mihari/version.rb +1 -1
  82. data/lib/mihari/web/app.rb +2 -1
  83. data/lib/mihari/web/controllers/analyzers_controller.rb +38 -0
  84. data/lib/mihari/web/controllers/base_controller.rb +1 -1
  85. data/lib/mihari/web/public/index.html +1 -21
  86. data/lib/mihari/web/public/redoc-static.html +338 -461
  87. data/lib/mihari/web/public/static/js/app.365f1907.js +13 -0
  88. data/lib/mihari/web/public/static/js/app.365f1907.js.map +1 -0
  89. data/lib/mihari/web/public/static/js/app.ab213f7c.js +12 -0
  90. data/lib/mihari/web/public/static/js/app.ab213f7c.js.map +1 -0
  91. data/mihari.gemspec +16 -9
  92. metadata +135 -58
  93. data/.rubocop.yml +0 -161
  94. data/lib/mihari/analyzers/free_text.rb +0 -48
  95. data/lib/mihari/analyzers/http_hash.rb +0 -100
  96. data/lib/mihari/analyzers/passive_dns.rb +0 -59
  97. data/lib/mihari/analyzers/passive_ssl.rb +0 -55
  98. data/lib/mihari/analyzers/reverse_whois.rb +0 -55
  99. data/lib/mihari/analyzers/securitytrails_domain_feed.rb +0 -59
  100. data/lib/mihari/analyzers/ssh_fingerprint.rb +0 -58
  101. data/lib/mihari/cli.rb +0 -126
  102. data/lib/mihari/commands/config.rb +0 -27
  103. data/lib/mihari/commands/free_text.rb +0 -21
  104. data/lib/mihari/commands/http_hash.rb +0 -25
  105. data/lib/mihari/commands/passive_dns.rb +0 -21
  106. data/lib/mihari/commands/passive_ssl.rb +0 -21
  107. data/lib/mihari/commands/reverse_whois.rb +0 -21
  108. data/lib/mihari/commands/securitytrails_domain_feed.rb +0 -23
  109. data/lib/mihari/commands/ssh_fingerprint.rb +0 -21
  110. data/lib/mihari/config.rb +0 -85
  111. data/lib/mihari/configurable.rb +0 -21
  112. data/lib/mihari/html.rb +0 -43
  113. data/lib/mihari/retriable.rb +0 -17
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "thor"
4
+
5
+ require "mihari/mixins/hash"
6
+
7
+ require "mihari/cli/mixins/utils"
8
+
9
+ module Mihari
10
+ module CLI
11
+ class Base < Thor
12
+ include Mihari::Mixins::Hash
13
+ include Mixins::Utils
14
+
15
+ class_option :config, type: :string, desc: "Path to the config file"
16
+
17
+ class_option :ignore_old_artifacts, type: :boolean, default: false, desc: "Whether to ignore old artifacts from checking or not. Only affects with analyze commands."
18
+ class_option :ignore_threshold, type: :numeric, default: 0, desc: "Number of days to define whether an artifact is old or not. Only affects with analyze commands."
19
+
20
+ class << self
21
+ def exit_on_failure?
22
+ true
23
+ end
24
+ end
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,13 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "thor"
4
+
5
+ require "mihari/commands/init"
6
+
7
+ module Mihari
8
+ module CLI
9
+ class Initialization < Base
10
+ include Mihari::Commands::Initialization
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,30 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Commands
4
+ require "mihari/commands/search"
5
+ require "mihari/commands/web"
6
+
7
+ # CLIs
8
+ require "mihari/cli/base"
9
+
10
+ require "mihari/cli/analyzer"
11
+ require "mihari/cli/init"
12
+ require "mihari/cli/validator"
13
+
14
+ module Mihari
15
+ module CLI
16
+ class Main < Base
17
+ include Mihari::Commands::Search
18
+ include Mihari::Commands::Web
19
+
20
+ desc "analyze", "Sub commands to run an analyzer"
21
+ subcommand "analyze", Analyzer
22
+
23
+ desc "init", "Sub commands to initialize config & rule"
24
+ subcommand "init", Initialization
25
+
26
+ desc "validate", "Sub commands to validate format of config & rule"
27
+ subcommand "validate", Validator
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,88 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Mihari
4
+ module CLI
5
+ module Mixins
6
+ module Utils
7
+ #
8
+ # Send an exception notification if there is any error in a block
9
+ #
10
+ # @return [Nil]
11
+ #
12
+ def with_error_handling
13
+ yield
14
+ rescue StandardError => e
15
+ notifier = Notifiers::ExceptionNotifier.new
16
+ notifier.notify e
17
+ end
18
+
19
+ #
20
+ # Check required keys in JSON
21
+ #
22
+ # @param [Hash] json
23
+ #
24
+ # @return [Boolean]
25
+ #
26
+ def required_alert_keys?(json)
27
+ %w[title description artifacts].all? { |key| json.key? key }
28
+ end
29
+
30
+ #
31
+ # Load configuration and establish DB connection
32
+ #
33
+ # @return [Hash]
34
+ #
35
+ def load_configuration
36
+ config = options["config"]
37
+ return unless config
38
+
39
+ Mihari.load_config_from_yaml config
40
+ Database.connect
41
+ end
42
+
43
+ #
44
+ # Run analyzer
45
+ #
46
+ # @param [Class<Mihari::Analyzers::Base>] analyzer_class
47
+ # @param [String] query
48
+ # @param [Hash] options
49
+ #
50
+ # @return [nil]
51
+ #
52
+ def run_analyzer(analyzer_class, query:, options:)
53
+ load_configuration
54
+
55
+ # options = Thor::CoreExt::HashWithIndifferentAccess
56
+ # ref. https://www.rubydoc.info/github/wycats/thor/Thor/CoreExt/HashWithIndifferentAccess
57
+ # so need to covert it to a plain hash
58
+ hash_options = options.to_hash
59
+
60
+ hash_options = symbolize_hash(hash_options)
61
+ hash_options = normalize_options(hash_options)
62
+
63
+ analyzer = analyzer_class.new(query, **hash_options)
64
+
65
+ analyzer.ignore_old_artifacts = options[:ignore_old_artifacts] || false
66
+ analyzer.ignore_threshold = options[:ignore_threshold] || 0
67
+
68
+ analyzer.run
69
+ end
70
+
71
+ #
72
+ # Normalize options (reject keys not for analyzers)
73
+ #
74
+ # @param [Hash] options
75
+ #
76
+ # @return [Hash]
77
+ #
78
+ def normalize_options(options)
79
+ # Delete :config because it is not intended to use for running an analyzer
80
+ [:config, :ignore_old_artifacts, :ignore_threshold].each do |ignore_key|
81
+ options.delete(ignore_key)
82
+ end
83
+ options
84
+ end
85
+ end
86
+ end
87
+ end
88
+ end
@@ -0,0 +1,11 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "mihari/commands/validator"
4
+
5
+ module Mihari
6
+ module CLI
7
+ class Validator < Base
8
+ include Mihari::Commands::Validator
9
+ end
10
+ end
11
+ end
@@ -5,7 +5,7 @@ module Mihari
5
5
  module BinaryEdge
6
6
  def self.included(thor)
7
7
  thor.class_eval do
8
- desc "binaryedge [QUERY]", "BinaryEdge host search by a query"
8
+ desc "binaryedge [QUERY]", "BinaryEdge host search"
9
9
  method_option :title, type: :string, desc: "title"
10
10
  method_option :description, type: :string, desc: "description"
11
11
  method_option :tags, type: :array, desc: "tags"
@@ -5,7 +5,7 @@ module Mihari
5
5
  module Censys
6
6
  def self.included(thor)
7
7
  thor.class_eval do
8
- desc "censys [QUERY]", "Censys IPv4 search by a query"
8
+ desc "censys [QUERY]", "Censys IPv4 search"
9
9
  method_option :title, type: :string, desc: "title"
10
10
  method_option :description, type: :string, desc: "description"
11
11
  method_option :tags, type: :array, desc: "tags"
@@ -5,13 +5,13 @@ module Mihari
5
5
  module CIRCL
6
6
  def self.included(thor)
7
7
  thor.class_eval do
8
- desc "circl [DOMAIN|SHA1]", "CIRCL passive DNS/SSL lookup by a domain or SHA1 certificate fingerprint"
8
+ desc "circl [DOMAIN|SHA1]", "CIRCL passive DNS/SSL search by a domain or SHA1 certificate fingerprint"
9
9
  method_option :title, type: :string, desc: "title"
10
10
  method_option :description, type: :string, desc: "description"
11
11
  method_option :tags, type: :array, desc: "tags"
12
12
  def circl(query)
13
13
  with_error_handling do
14
- run_analyzer Analyzers::CIRCL, query: refang(query), options: options
14
+ run_analyzer Analyzers::CIRCL, query: query, options: options
15
15
  end
16
16
  end
17
17
  end
@@ -5,7 +5,7 @@ module Mihari
5
5
  module Crtsh
6
6
  def self.included(thor)
7
7
  thor.class_eval do
8
- desc "crtsh [QUERY]", "crt.sh search by a query"
8
+ desc "crtsh [QUERY]", "crt.sh search"
9
9
  method_option :title, type: :string, desc: "title"
10
10
  method_option :description, type: :string, desc: "description"
11
11
  method_option :tags, type: :array, desc: "tags"
@@ -5,7 +5,7 @@ module Mihari
5
5
  module DNPedia
6
6
  def self.included(thor)
7
7
  thor.class_eval do
8
- desc "dnpedia [QUERY]", "DNPedia domain search by a query"
8
+ desc "dnpedia [QUERY]", "DNPedia domain search"
9
9
  method_option :title, type: :string, desc: "title"
10
10
  method_option :description, type: :string, desc: "description"
11
11
  method_option :tags, type: :array, desc: "tags"
@@ -5,13 +5,13 @@ module Mihari
5
5
  module DNSTwister
6
6
  def self.included(thor)
7
7
  thor.class_eval do
8
- desc "dnstwister [DOMAIN]", "dnstwister lookup by a domain"
8
+ desc "dnstwister [DOMAIN]", "dnstwister search"
9
9
  method_option :title, type: :string, desc: "title"
10
10
  method_option :description, type: :string, desc: "description"
11
11
  method_option :tags, type: :array, desc: "tags"
12
12
  def dnstwister(domain)
13
13
  with_error_handling do
14
- run_analyzer Analyzers::DNSTwister, query: refang(domain), options: options
14
+ run_analyzer Analyzers::DNSTwister, query: domain, options: options
15
15
  end
16
16
  end
17
17
  end
@@ -0,0 +1,46 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "colorize"
4
+
5
+ module Mihari
6
+ module Commands
7
+ module Initialization
8
+ def self.included(thor)
9
+ include Mixins::Configuration
10
+ include Mixins::Rule
11
+
12
+ thor.class_eval do
13
+ desc "config", "Create a config file"
14
+ method_option :filename, type: :string, default: "mihari.yml"
15
+ def config
16
+ filename = options["filename"]
17
+
18
+ warning = "#{filename} exists. Do you want to overwrite it? (y/n)"
19
+ if File.exist?(filename) && !(yes? warning)
20
+ return
21
+ end
22
+
23
+ initialize_config_yaml filename
24
+
25
+ puts "The config file is initialized as #{filename}.".colorize(:blue)
26
+ end
27
+
28
+ desc "rule", "Create a rule file"
29
+ method_option :filename, type: :string, default: "rule.yml"
30
+ def rule
31
+ filename = options["filename"]
32
+
33
+ warning = "#{filename} exists. Do you want to overwrite it? (y/n)"
34
+ if File.exist?(filename) && !(yes? warning)
35
+ return
36
+ end
37
+
38
+ initialize_rule_yaml filename
39
+
40
+ puts "The rule file is created as #{filename}.".colorize(:blue)
41
+ end
42
+ end
43
+ end
44
+ end
45
+ end
46
+ end
@@ -12,7 +12,7 @@ module Mihari
12
12
  raise ArgumentError, "Input not found: please give an input in a JSON format" unless json
13
13
 
14
14
  json = parse_as_json(json)
15
- raise ArgumentError, "Invalid input format: an input JSON data should have title, description and artifacts key" unless valid_json?(json)
15
+ raise ArgumentError, "Invalid input format: an input JSON data should have title, description and artifacts key" unless required_alert_keys?(json)
16
16
 
17
17
  title = json["title"]
18
18
  description = json["description"]
@@ -5,7 +5,7 @@ module Mihari
5
5
  module Onyphe
6
6
  def self.included(thor)
7
7
  thor.class_eval do
8
- desc "onyphe [QUERY]", "Onyphe datascan search by a query"
8
+ desc "onyphe [QUERY]", "Onyphe datascan search"
9
9
  method_option :title, type: :string, desc: "title"
10
10
  method_option :description, type: :string, desc: "description"
11
11
  method_option :tags, type: :array, desc: "tags"
@@ -5,13 +5,13 @@ module Mihari
5
5
  module OTX
6
6
  def self.included(thor)
7
7
  thor.class_eval do
8
- desc "otx [IP|DOMAIN]", "OTX lookup by an IP or domain"
8
+ desc "otx [IP|DOMAIN]", "OTX search by an IP or domain"
9
9
  method_option :title, type: :string, desc: "title"
10
10
  method_option :description, type: :string, desc: "description"
11
11
  method_option :tags, type: :array, desc: "tags"
12
12
  def otx(domain)
13
13
  with_error_handling do
14
- run_analyzer Analyzers::OTX, query: refang(domain), options: options
14
+ run_analyzer Analyzers::OTX, query: domain, options: options
15
15
  end
16
16
  end
17
17
  end
@@ -5,13 +5,13 @@ module Mihari
5
5
  module PassiveTotal
6
6
  def self.included(thor)
7
7
  thor.class_eval do
8
- desc "passivetotal [IP|DOMAIN|EMAIL|SHA1]", "PassiveTotal lookup by an ip, domain, email or SHA1 certificate fingerprint"
8
+ desc "passivetotal [IP|DOMAIN|EMAIL|SHA1]", "PassiveTotal search by an ip, domain, email or SHA1 certificate fingerprint"
9
9
  method_option :title, type: :string, desc: "title"
10
10
  method_option :description, type: :string, desc: "description"
11
11
  method_option :tags, type: :array, desc: "tags"
12
12
  def passivetotal(indicator)
13
13
  with_error_handling do
14
- run_analyzer Analyzers::PassiveTotal, query: refang(indicator), options: options
14
+ run_analyzer Analyzers::PassiveTotal, query: indicator, options: options
15
15
  end
16
16
  end
17
17
  end
@@ -5,13 +5,13 @@ module Mihari
5
5
  module Pulsedive
6
6
  def self.included(thor)
7
7
  thor.class_eval do
8
- desc "pulsedive [IP|DOMAIN]", "Pulsedive lookup by an ip or domain"
8
+ desc "pulsedive [IP|DOMAIN]", "Pulsedive search by an ip or domain"
9
9
  method_option :title, type: :string, desc: "title"
10
10
  method_option :description, type: :string, desc: "description"
11
11
  method_option :tags, type: :array, desc: "tags"
12
12
  def pulsedive(indiactor)
13
13
  with_error_handling do
14
- run_analyzer Analyzers::Pulsedive, query: refang(indiactor), options: options
14
+ run_analyzer Analyzers::Pulsedive, query: indiactor, options: options
15
15
  end
16
16
  end
17
17
  end
@@ -0,0 +1,77 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Mihari
4
+ module Commands
5
+ module Search
6
+ include Mixins::Rule
7
+
8
+ def self.included(thor)
9
+ thor.class_eval do
10
+ desc "search [RULE]", "Search by a rule"
11
+ def search_by_rule(rule)
12
+ # convert str(YAML) to hash or str(path/YAML file) to hash
13
+ rule = load_rule(rule)
14
+
15
+ # validate rule schema
16
+ validate_rule rule
17
+
18
+ analyzer = build_rule_analyzer(**rule)
19
+
20
+ ignore_old_artifacts = options["ignore_old_artifacts"] || false
21
+ ignore_threshold = options["ignore_threshold"] || 0
22
+
23
+ with_error_handling do
24
+ run_rule_analyzer analyzer, ignore_old_artifacts: ignore_old_artifacts, ignore_threshold: ignore_threshold
25
+ end
26
+ end
27
+ end
28
+ end
29
+
30
+ private
31
+
32
+ #
33
+ # Build a rule analyzer
34
+ #
35
+ # @param [String] title
36
+ # @param [String] description
37
+ # @param [Array<Hash>] queries
38
+ # @param [Array<String>, nil] tags
39
+ # @param [Array<String>, nil] allowed_data_types
40
+ # @param [String, nil] source
41
+ #
42
+ # @return [Mihari::Analyzers::Rule]
43
+ #
44
+ def build_rule_analyzer(title:, description:, queries:, tags: nil, allowed_data_types: nil, source: nil)
45
+ tags = [] if tags.nil?
46
+ allowed_data_types = ALLOWED_DATA_TYPES if allowed_data_types.nil?
47
+
48
+ Analyzers::Rule.new(
49
+ title: title,
50
+ description: description,
51
+ tags: tags,
52
+ queries: queries,
53
+ allowed_data_types: allowed_data_types,
54
+ source: source
55
+ )
56
+ end
57
+
58
+ #
59
+ # Run rule analyzer
60
+ #
61
+ # @param [Mihari::Analyzer::Rule] analyzer
62
+ # @param [Boolean] ignore_old_artifacts
63
+ # @param [Integer] ignore_threshold
64
+ #
65
+ # @return [nil]
66
+ #
67
+ def run_rule_analyzer(analyzer, ignore_old_artifacts: false, ignore_threshold: 0)
68
+ load_configuration
69
+
70
+ analyzer.ignore_old_artifacts = ignore_old_artifacts
71
+ analyzer.ignore_threshold = ignore_threshold
72
+
73
+ analyzer.run
74
+ end
75
+ end
76
+ end
77
+ end