mihari 1.5.1 → 2.3.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (93) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE/bug_report.md +43 -0
  3. data/.github/ISSUE_TEMPLATE/feature_request.md +15 -0
  4. data/.github/workflows/test.yml +41 -17
  5. data/.rubocop.yml +6 -0
  6. data/.standard.yml +4 -0
  7. data/README.md +23 -274
  8. data/bin/console +1 -0
  9. data/build_frontend.sh +14 -0
  10. data/docker/Dockerfile +3 -2
  11. data/examples/ipinfo_hosted_domains.rb +1 -1
  12. data/images/{eyecatch.png → overview.png} +0 -0
  13. data/images/tines.png +0 -0
  14. data/images/web_alerts.png +0 -0
  15. data/images/web_config.png +0 -0
  16. data/lib/mihari.rb +2 -2
  17. data/lib/mihari/analyzers/base.rb +11 -2
  18. data/lib/mihari/analyzers/circl.rb +3 -3
  19. data/lib/mihari/analyzers/onyphe.rb +2 -2
  20. data/lib/mihari/analyzers/securitytrails.rb +2 -2
  21. data/lib/mihari/analyzers/urlscan.rb +1 -6
  22. data/lib/mihari/cli.rb +73 -301
  23. data/lib/mihari/commands/binaryedge.rb +21 -0
  24. data/lib/mihari/commands/censys.rb +22 -0
  25. data/lib/mihari/commands/circl.rb +21 -0
  26. data/lib/mihari/commands/config.rb +27 -0
  27. data/lib/mihari/commands/crtsh.rb +22 -0
  28. data/lib/mihari/commands/dnpedia.rb +21 -0
  29. data/lib/mihari/commands/dnstwister.rb +21 -0
  30. data/lib/mihari/commands/free_text.rb +21 -0
  31. data/lib/mihari/commands/http_hash.rb +25 -0
  32. data/lib/mihari/commands/json.rb +42 -0
  33. data/lib/mihari/commands/onyphe.rb +21 -0
  34. data/lib/mihari/commands/otx.rb +21 -0
  35. data/lib/mihari/commands/passive_dns.rb +21 -0
  36. data/lib/mihari/commands/passive_ssl.rb +21 -0
  37. data/lib/mihari/commands/passivetotal.rb +21 -0
  38. data/lib/mihari/commands/pulsedive.rb +21 -0
  39. data/lib/mihari/commands/reverse_whois.rb +21 -0
  40. data/lib/mihari/commands/securitytrails.rb +22 -0
  41. data/lib/mihari/commands/securitytrails_domain_feed.rb +23 -0
  42. data/lib/mihari/commands/shodan.rb +21 -0
  43. data/lib/mihari/commands/spyse.rb +22 -0
  44. data/lib/mihari/commands/ssh_fingerprint.rb +21 -0
  45. data/lib/mihari/commands/urlscan.rb +23 -0
  46. data/lib/mihari/commands/virustotal.rb +21 -0
  47. data/lib/mihari/commands/web.rb +22 -0
  48. data/lib/mihari/commands/zoomeye.rb +22 -0
  49. data/lib/mihari/config.rb +12 -0
  50. data/lib/mihari/configurable.rb +4 -5
  51. data/lib/mihari/database.rb +8 -2
  52. data/lib/mihari/emitters/slack.rb +1 -1
  53. data/lib/mihari/emitters/the_hive.rb +1 -1
  54. data/lib/mihari/models/alert.rb +51 -0
  55. data/lib/mihari/models/artifact.rb +13 -2
  56. data/lib/mihari/notifiers/exception_notifier.rb +4 -4
  57. data/lib/mihari/serializers/alert.rb +1 -1
  58. data/lib/mihari/serializers/artifact.rb +1 -1
  59. data/lib/mihari/serializers/tag.rb +1 -1
  60. data/lib/mihari/status.rb +6 -14
  61. data/lib/mihari/version.rb +1 -1
  62. data/lib/mihari/web/app.rb +51 -0
  63. data/lib/mihari/web/controllers/alerts_controller.rb +75 -0
  64. data/lib/mihari/web/controllers/artifacts_controller.rb +24 -0
  65. data/lib/mihari/web/controllers/base_controller.rb +22 -0
  66. data/lib/mihari/web/controllers/command_controller.rb +26 -0
  67. data/lib/mihari/web/controllers/config_controller.rb +13 -0
  68. data/lib/mihari/web/controllers/sources_controller.rb +12 -0
  69. data/lib/mihari/web/controllers/tags_controller.rb +28 -0
  70. data/lib/mihari/web/helpers/json.rb +53 -0
  71. data/lib/mihari/web/public/index.html +21 -0
  72. data/lib/mihari/web/public/redoc-static.html +519 -0
  73. data/lib/mihari/web/public/static/favicon.ico +0 -0
  74. data/lib/mihari/web/public/static/fonts/fa-brands-400.099a9556.woff +0 -0
  75. data/lib/mihari/web/public/static/fonts/fa-brands-400.30cc681d.eot +0 -0
  76. data/lib/mihari/web/public/static/fonts/fa-brands-400.3b89dd10.ttf +0 -0
  77. data/lib/mihari/web/public/static/fonts/fa-brands-400.f7307680.woff2 +0 -0
  78. data/lib/mihari/web/public/static/fonts/fa-regular-400.1f77739c.ttf +0 -0
  79. data/lib/mihari/web/public/static/fonts/fa-regular-400.7124eb50.woff +0 -0
  80. data/lib/mihari/web/public/static/fonts/fa-regular-400.7630483d.eot +0 -0
  81. data/lib/mihari/web/public/static/fonts/fa-regular-400.f0f82301.woff2 +0 -0
  82. data/lib/mihari/web/public/static/fonts/fa-solid-900.1042e8ca.eot +0 -0
  83. data/lib/mihari/web/public/static/fonts/fa-solid-900.605ed792.ttf +0 -0
  84. data/lib/mihari/web/public/static/fonts/fa-solid-900.9fe5a17c.woff +0 -0
  85. data/lib/mihari/web/public/static/fonts/fa-solid-900.e8a427e1.woff2 +0 -0
  86. data/lib/mihari/web/public/static/img/fa-brands-400.ba7ed552.svg +3717 -0
  87. data/lib/mihari/web/public/static/img/fa-regular-400.0bb42845.svg +801 -0
  88. data/lib/mihari/web/public/static/img/fa-solid-900.376c1f97.svg +5034 -0
  89. data/lib/mihari/web/public/static/js/app.cccddb2b.js +12 -0
  90. data/lib/mihari/web/public/static/js/app.cccddb2b.js.map +1 -0
  91. data/mihari.gemspec +16 -4
  92. metadata +261 -18
  93. data/lib/mihari/alert_viewer.rb +0 -23
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Mihari
4
+ module Commands
5
+ module SecurityTrailsDomainFeed
6
+ def self.included(thor)
7
+ thor.class_eval do
8
+ desc "securitytrails_domain_feed [REGEXP]", "SecurityTrails new domain feed search by a regexp"
9
+ method_option :title, type: :string, desc: "title"
10
+ method_option :description, type: :string, desc: "description"
11
+ method_option :tags, type: :array, desc: "tags"
12
+ method_option :type, type: :string, default: "registered", desc: "A type of domain feed ('all', 'new' or 'registered')"
13
+ def securitytrails_domain_feed(regexp)
14
+ with_error_handling do
15
+ run_analyzer Analyzers::SecurityTrailsDomainFeed, query: regexp, options: options
16
+ end
17
+ end
18
+ map "st_domain_feed" => :securitytrails_domain_feedd
19
+ end
20
+ end
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,21 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Mihari
4
+ module Commands
5
+ module Shodan
6
+ def self.included(thor)
7
+ thor.class_eval do
8
+ desc "shodan [QUERY]", "Shodan host search by a query"
9
+ method_option :title, type: :string, desc: "title"
10
+ method_option :description, type: :string, desc: "description"
11
+ method_option :tags, type: :array, desc: "tags"
12
+ def shodan(query)
13
+ with_error_handling do
14
+ run_analyzer Analyzers::Shodan, query: query, options: options
15
+ end
16
+ end
17
+ end
18
+ end
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Mihari
4
+ module Commands
5
+ module Spyse
6
+ def self.included(thor)
7
+ thor.class_eval do
8
+ desc "spyse [QUERY]", "Spyse search by a query"
9
+ method_option :title, type: :string, desc: "title"
10
+ method_option :description, type: :string, desc: "description"
11
+ method_option :tags, type: :array, desc: "tags"
12
+ method_option :type, type: :string, desc: "type to search (ip or domain)", default: "doamin"
13
+ def spyse(query)
14
+ with_error_handling do
15
+ run_analyzer Analyzers::Spyse, query: query, options: options
16
+ end
17
+ end
18
+ end
19
+ end
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,21 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Mihari
4
+ module Commands
5
+ module SSHFingerprint
6
+ def self.included(thor)
7
+ thor.class_eval do
8
+ desc "ssh_fingerprint [FINGERPRINT]", "Cross search with search engines by an SSH fingerprint (e.g. dc:14:de:8e:d7:c1:15:43:23:82:25:81:d2:59:e8:c0)"
9
+ method_option :title, type: :string, desc: "title"
10
+ method_option :description, type: :string, desc: "description"
11
+ method_option :tags, type: :array, desc: "tags"
12
+ def ssh_fingerprint(fingerprint)
13
+ with_error_handling do
14
+ run_analyzer Analyzers::SSHFingerprint, query: fingerprint, options: options
15
+ end
16
+ end
17
+ end
18
+ end
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Mihari
4
+ module Commands
5
+ module Urlscan
6
+ def self.included(thor)
7
+ thor.class_eval do
8
+ desc "urlscan [QUERY]", "urlscan search by a given query"
9
+ method_option :title, type: :string, desc: "title"
10
+ method_option :description, type: :string, desc: "description"
11
+ method_option :tags, type: :array, desc: "tags"
12
+ method_option :target_type, type: :string, default: "url", desc: "target type to fetch from lookup results (target type should be 'url', 'domain' or 'ip')"
13
+ method_option :use_similarity, type: :boolean, default: false, desc: "use similarity API or not"
14
+ def urlscan(query)
15
+ with_error_handling do
16
+ run_analyzer Analyzers::Urlscan, query: query, options: options
17
+ end
18
+ end
19
+ end
20
+ end
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,21 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Mihari
4
+ module Commands
5
+ module VirusTotal
6
+ def self.included(thor)
7
+ thor.class_eval do
8
+ desc "virustotal [IP|DOMAIN]", "VirusTotal resolutions lookup by an ip or domain"
9
+ method_option :title, type: :string, desc: "title"
10
+ method_option :description, type: :string, desc: "description"
11
+ method_option :tags, type: :array, desc: "tags"
12
+ def virustotal(indiactor)
13
+ with_error_handling do
14
+ run_analyzer Analyzers::VirusTotal, query: refang(indiactor), options: options
15
+ end
16
+ end
17
+ end
18
+ end
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Mihari
4
+ module Commands
5
+ module Web
6
+ def self.included(thor)
7
+ thor.class_eval do
8
+ desc "web", "Launch the web app"
9
+ method_option :port, type: :numeric, default: 9292
10
+ method_option :host, type: :string, default: "localhost"
11
+ def web
12
+ port = options["port"].to_i || 9292
13
+ host = options["host"] || "localhost"
14
+
15
+ load_configuration
16
+ Mihari::App.run!(port: port, host: host)
17
+ end
18
+ end
19
+ end
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Mihari
4
+ module Commands
5
+ module ZoomEye
6
+ def self.included(thor)
7
+ thor.class_eval do
8
+ desc "zoomeye [QUERY]", "ZoomEye search by a query"
9
+ method_option :title, type: :string, desc: "title"
10
+ method_option :description, type: :string, desc: "description"
11
+ method_option :tags, type: :array, desc: "tags"
12
+ method_option :type, type: :string, desc: "type to search(host / web)", default: "host"
13
+ def zoomeye(query)
14
+ with_error_handling do
15
+ run_analyzer Analyzers::ZoomEye, query: query, options: options
16
+ end
17
+ end
18
+ end
19
+ end
20
+ end
21
+ end
22
+ end
data/lib/mihari/config.rb CHANGED
@@ -55,6 +55,18 @@ module Mihari
55
55
  end
56
56
  end
57
57
  end
58
+
59
+ def initialize_yaml(filename)
60
+ keys = new.instance_variables.map do |key|
61
+ key.to_s[1..]
62
+ end
63
+
64
+ config = keys.map do |key|
65
+ [key, nil]
66
+ end.to_h
67
+
68
+ YAML.dump(config, File.open(filename, "w"))
69
+ end
58
70
  end
59
71
  end
60
72
 
@@ -6,13 +6,12 @@ module Mihari
6
6
  config_keys.all? { |key| Mihari.config.send(key) }
7
7
  end
8
8
 
9
- def configuration_status
9
+ def configuration_values
10
10
  return nil if config_keys.empty?
11
11
 
12
- names = config_keys.join(" and ")
13
- be_verb = config_keys.length == 1 ? "is" : "are"
14
- status = configured? ? "found" : "missing"
15
- "#{names} #{be_verb} #{status}"
12
+ config_keys.map do |key|
13
+ { key: key.upcase, value: Mihari.config.send(key) }
14
+ end
16
15
  end
17
16
 
18
17
  def config_keys
@@ -34,6 +34,7 @@ end
34
34
 
35
35
  def adapter
36
36
  return "postgresql" if Mihari.config.database.start_with?("postgresql://", "postgres://")
37
+ return "mysql2" if Mihari.config.database.start_with?("mysql2://")
37
38
 
38
39
  "sqlite3"
39
40
  end
@@ -43,7 +44,7 @@ module Mihari
43
44
  class << self
44
45
  def connect
45
46
  case adapter
46
- when "postgresql"
47
+ when "postgresql", "mysql2"
47
48
  ActiveRecord::Base.establish_connection(Mihari.config.database)
48
49
  else
49
50
  ActiveRecord::Base.establish_connection(
@@ -54,10 +55,15 @@ module Mihari
54
55
 
55
56
  ActiveRecord::Migration.verbose = false
56
57
  InitialSchema.migrate(:up)
57
- rescue
58
+ rescue StandardError
58
59
  # Do nothing
59
60
  end
60
61
 
62
+ def close
63
+ ActiveRecord::Base.clear_active_connections!
64
+ ActiveRecord::Base.connection.close
65
+ end
66
+
61
67
  def destroy!
62
68
  InitialSchema.migrate(:down) if ActiveRecord::Base.connected?
63
69
  end
@@ -132,7 +132,7 @@ module Mihari
132
132
  [
133
133
  "*#{title}*",
134
134
  "*Desc.*: #{description}",
135
- "*Tags*: #{tags.join(', ')}"
135
+ "*Tags*: #{tags.join(", ")}"
136
136
  ].join("\n")
137
137
  end
138
138
 
@@ -17,7 +17,7 @@ module Mihari
17
17
  api.alert.create(
18
18
  title: title,
19
19
  description: description,
20
- artifacts: artifacts.map { |artifact| {data: artifact.data, data_type: artifact.data_type, message: description} },
20
+ artifacts: artifacts.map { |artifact| { data: artifact.data, data_type: artifact.data_type, message: description } },
21
21
  tags: tags,
22
22
  type: "external",
23
23
  source: "mihari"
@@ -1,11 +1,62 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require "active_record"
4
+ require "active_record/filter"
4
5
 
5
6
  module Mihari
6
7
  class Alert < ActiveRecord::Base
7
8
  has_many :taggings, dependent: :destroy
8
9
  has_many :artifacts, dependent: :destroy
9
10
  has_many :tags, through: :taggings
11
+
12
+ class << self
13
+ def search(artifact_data: nil, description: nil, source: nil, tag_name: nil, title: nil, from_at: nil, to_at: nil, limit: 10, page: 1)
14
+ limit = limit.to_i
15
+ raise ArgumentError, "limit should be bigger than zero" unless limit.positive?
16
+
17
+ page = page.to_i
18
+ raise ArgumentError, "page should be bigger than zero" unless page.positive?
19
+
20
+ offset = (page - 1) * limit
21
+
22
+ relation = build_relation(artifact_data: artifact_data, title: title, description: description, source: source, tag_name: tag_name, from_at: from_at, to_at: to_at)
23
+ # relation = relation.group("alerts.id")
24
+
25
+ alerts = relation.limit(limit).offset(offset).order(id: :desc)
26
+
27
+ alerts.map do |alert|
28
+ json = AlertSerializer.new(alert).as_json
29
+ json[:artifacts] = json[:artifacts] || []
30
+ json[:tags] = json[:tags] || []
31
+ json
32
+ end
33
+ end
34
+
35
+ def count(artifact_data: nil, description: nil, source: nil, tag_name: nil, title: nil, from_at: nil, to_at: nil)
36
+ relation = build_relation(artifact_data: artifact_data, title: title, description: description, source: source, tag_name: tag_name, from_at: from_at, to_at: to_at)
37
+ relation.distinct("alerts.id").count
38
+ end
39
+
40
+ private
41
+
42
+ def build_relation(artifact_data: nil, title: nil, description: nil, source: nil, tag_name: nil, from_at: nil, to_at: nil)
43
+ relation = self
44
+ relation = joins(:tags) if tag_name
45
+ relation = joins(:artifacts) if artifact_data
46
+
47
+ relation = relation.where(artifacts: { data: artifact_data }) if artifact_data
48
+ relation = relation.where(tags: { name: tag_name }) if tag_name
49
+
50
+ relation = relation.where(source: source) if source
51
+ relation = relation.where(title: title) if title
52
+
53
+ relation = relation.filter(description: { like: "%#{description}%" }) if description
54
+
55
+ relation = relation.filter(created_at: { gte: from_at }) if from_at
56
+ relation = relation.filter(created_at: { lte: to_at }) if to_at
57
+
58
+ relation
59
+ end
60
+ end
10
61
  end
11
62
  end
@@ -1,6 +1,9 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require "active_record"
4
+ require "active_record/filter"
5
+ require "active_support/core_ext/integer/time"
6
+ require "active_support/core_ext/numeric/time"
4
7
 
5
8
  class ArtifactValidator < ActiveModel::Validator
6
9
  def validate(record)
@@ -20,8 +23,16 @@ module Mihari
20
23
  self.data_type = TypeChecker.type(data)
21
24
  end
22
25
 
23
- def unique?
24
- self.class.find_by(data: data).nil?
26
+ def unique?(ignore_old_artifacts: false, ignore_threshold: 0)
27
+ artifact = self.class.where(data: data).order(created_at: :desc).first
28
+ return true if artifact.nil?
29
+
30
+ return false unless ignore_old_artifacts
31
+
32
+ days_before = (-ignore_threshold).days.from_now
33
+ # if an artifact is created before {ignore_threshold} days, ignore it
34
+ # within {ignore_threshold} days, do not ignore it
35
+ artifact.created_at < days_before
25
36
  end
26
37
  end
27
38
  end
@@ -51,20 +51,20 @@ module Mihari
51
51
 
52
52
  def to_fields(clean_message, backtrace)
53
53
  fields = [
54
- {title: "Exception", value: clean_message},
55
- {title: "Hostname", value: hostname}
54
+ { title: "Exception", value: clean_message },
55
+ { title: "Hostname", value: hostname }
56
56
  ]
57
57
 
58
58
  if backtrace
59
59
  formatted_backtrace = format_backtrace(backtrace)
60
- fields << {title: "Backtrace", value: formatted_backtrace}
60
+ fields << { title: "Backtrace", value: formatted_backtrace }
61
61
  end
62
62
  fields
63
63
  end
64
64
 
65
65
  def hostname
66
66
  Socket.gethostname
67
- rescue => _e
67
+ rescue StandardError => _e
68
68
  "N/A"
69
69
  end
70
70
 
@@ -4,7 +4,7 @@ require "active_model_serializers"
4
4
 
5
5
  module Mihari
6
6
  class AlertSerializer < ActiveModel::Serializer
7
- attributes :title, :description, :source, :created_at
7
+ attributes :id, :title, :description, :source, :created_at
8
8
 
9
9
  has_many :artifacts
10
10
  has_many :tags, through: :taggings
@@ -4,6 +4,6 @@ require "active_model_serializers"
4
4
 
5
5
  module Mihari
6
6
  class ArtifactSerializer < ActiveModel::Serializer
7
- attributes :data, :data_type
7
+ attributes :id, :data, :data_type
8
8
  end
9
9
  end
@@ -4,6 +4,6 @@ require "active_model_serializers"
4
4
 
5
5
  module Mihari
6
6
  class TagSerializer < ActiveModel::Serializer
7
- attributes :name
7
+ attributes :id, :name
8
8
  end
9
9
  end
data/lib/mihari/status.rb CHANGED
@@ -3,9 +3,7 @@
3
3
  module Mihari
4
4
  class Status
5
5
  def check
6
- statuses.map do |key, value|
7
- [key, convert(**value)]
8
- end.to_h
6
+ statuses
9
7
  end
10
8
 
11
9
  def self.check
@@ -14,16 +12,9 @@ module Mihari
14
12
 
15
13
  private
16
14
 
17
- def convert(status:, message:)
18
- {
19
- status: status ? "OK" : "Bad",
20
- message: message
21
- }
22
- end
23
-
24
15
  def statuses
25
16
  (Mihari.analyzers + Mihari.emitters).map do |klass|
26
- name = klass.to_s.downcase.split("::").last.to_s
17
+ name = klass.to_s.split("::").last.to_s
27
18
 
28
19
  [name, build_status(klass)]
29
20
  end.to_h.compact
@@ -33,10 +24,11 @@ module Mihari
33
24
  is_analyzer = klass.ancestors.include?(Mihari::Analyzers::Base)
34
25
 
35
26
  instance = is_analyzer ? klass.new("dummy") : klass.new
36
- status = instance.configured?
37
- message = instance.configuration_status
27
+ is_configured = instance.configured?
28
+ values = instance.configuration_values
29
+ type = is_analyzer ? "Analyzer" : "Emitter"
38
30
 
39
- message ? {status: status, message: message} : nil
31
+ values ? { is_configured: is_configured, values: values, type: type } : nil
40
32
  rescue ArgumentError => _e
41
33
  nil
42
34
  end