mihari 2.0.0 → 2.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE/bug_report.md +43 -0
  3. data/.github/ISSUE_TEMPLATE/feature_request.md +15 -0
  4. data/.rubocop.yml +6 -0
  5. data/.standard.yml +4 -0
  6. data/README.md +11 -1
  7. data/bin/console +1 -0
  8. data/docker/Dockerfile +3 -2
  9. data/examples/ipinfo_hosted_domains.rb +1 -1
  10. data/images/{eyecatch.png → overview.png} +0 -0
  11. data/images/tines.png +0 -0
  12. data/images/web_alerts.png +0 -0
  13. data/images/web_config.png +0 -0
  14. data/lib/mihari/analyzers/base.rb +11 -2
  15. data/lib/mihari/analyzers/circl.rb +3 -3
  16. data/lib/mihari/analyzers/onyphe.rb +2 -2
  17. data/lib/mihari/analyzers/securitytrails.rb +2 -2
  18. data/lib/mihari/analyzers/urlscan.rb +1 -6
  19. data/lib/mihari/analyzers/zoomeye.rb +2 -2
  20. data/lib/mihari/cli.rb +72 -289
  21. data/lib/mihari/commands/binaryedge.rb +21 -0
  22. data/lib/mihari/commands/censys.rb +22 -0
  23. data/lib/mihari/commands/circl.rb +21 -0
  24. data/lib/mihari/commands/config.rb +27 -0
  25. data/lib/mihari/commands/crtsh.rb +22 -0
  26. data/lib/mihari/commands/dnpedia.rb +21 -0
  27. data/lib/mihari/commands/dnstwister.rb +21 -0
  28. data/lib/mihari/commands/free_text.rb +21 -0
  29. data/lib/mihari/commands/http_hash.rb +25 -0
  30. data/lib/mihari/commands/json.rb +42 -0
  31. data/lib/mihari/commands/onyphe.rb +21 -0
  32. data/lib/mihari/commands/otx.rb +21 -0
  33. data/lib/mihari/commands/passive_dns.rb +21 -0
  34. data/lib/mihari/commands/passive_ssl.rb +21 -0
  35. data/lib/mihari/commands/passivetotal.rb +21 -0
  36. data/lib/mihari/commands/pulsedive.rb +21 -0
  37. data/lib/mihari/commands/reverse_whois.rb +21 -0
  38. data/lib/mihari/commands/securitytrails.rb +22 -0
  39. data/lib/mihari/commands/securitytrails_domain_feed.rb +23 -0
  40. data/lib/mihari/commands/shodan.rb +21 -0
  41. data/lib/mihari/commands/spyse.rb +22 -0
  42. data/lib/mihari/commands/ssh_fingerprint.rb +21 -0
  43. data/lib/mihari/commands/urlscan.rb +23 -0
  44. data/lib/mihari/commands/virustotal.rb +21 -0
  45. data/lib/mihari/commands/web.rb +22 -0
  46. data/lib/mihari/commands/zoomeye.rb +22 -0
  47. data/lib/mihari/config.rb +14 -3
  48. data/lib/mihari/configurable.rb +1 -1
  49. data/lib/mihari/emitters/slack.rb +4 -4
  50. data/lib/mihari/emitters/the_hive.rb +1 -1
  51. data/lib/mihari/models/alert.rb +5 -5
  52. data/lib/mihari/models/artifact.rb +13 -2
  53. data/lib/mihari/notifiers/exception_notifier.rb +4 -4
  54. data/lib/mihari/status.rb +2 -10
  55. data/lib/mihari/version.rb +1 -1
  56. data/lib/mihari/web/app.rb +25 -100
  57. data/lib/mihari/web/controllers/alerts_controller.rb +75 -0
  58. data/lib/mihari/web/controllers/artifacts_controller.rb +24 -0
  59. data/lib/mihari/web/controllers/base_controller.rb +22 -0
  60. data/lib/mihari/web/controllers/command_controller.rb +26 -0
  61. data/lib/mihari/web/controllers/config_controller.rb +13 -0
  62. data/lib/mihari/web/controllers/sources_controller.rb +12 -0
  63. data/lib/mihari/web/controllers/tags_controller.rb +28 -0
  64. data/lib/mihari/web/helpers/json.rb +53 -0
  65. data/lib/mihari/web/public/index.html +2 -2
  66. data/lib/mihari/web/public/redoc-static.html +519 -0
  67. data/lib/mihari/web/public/static/js/{app.58b32d15.js → app.cccddb2b.js} +4 -4
  68. data/lib/mihari/web/public/static/js/app.cccddb2b.js.map +1 -0
  69. data/mihari.gemspec +9 -3
  70. metadata +146 -23
  71. data/lib/mihari/web/public/static/js/app.58b32d15.js.map +0 -1
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Mihari
4
+ module Commands
5
+ module SecurityTrailsDomainFeed
6
+ def self.included(thor)
7
+ thor.class_eval do
8
+ desc "securitytrails_domain_feed [REGEXP]", "SecurityTrails new domain feed search by a regexp"
9
+ method_option :title, type: :string, desc: "title"
10
+ method_option :description, type: :string, desc: "description"
11
+ method_option :tags, type: :array, desc: "tags"
12
+ method_option :type, type: :string, default: "registered", desc: "A type of domain feed ('all', 'new' or 'registered')"
13
+ def securitytrails_domain_feed(regexp)
14
+ with_error_handling do
15
+ run_analyzer Analyzers::SecurityTrailsDomainFeed, query: regexp, options: options
16
+ end
17
+ end
18
+ map "st_domain_feed" => :securitytrails_domain_feedd
19
+ end
20
+ end
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,21 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Mihari
4
+ module Commands
5
+ module Shodan
6
+ def self.included(thor)
7
+ thor.class_eval do
8
+ desc "shodan [QUERY]", "Shodan host search by a query"
9
+ method_option :title, type: :string, desc: "title"
10
+ method_option :description, type: :string, desc: "description"
11
+ method_option :tags, type: :array, desc: "tags"
12
+ def shodan(query)
13
+ with_error_handling do
14
+ run_analyzer Analyzers::Shodan, query: query, options: options
15
+ end
16
+ end
17
+ end
18
+ end
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Mihari
4
+ module Commands
5
+ module Spyse
6
+ def self.included(thor)
7
+ thor.class_eval do
8
+ desc "spyse [QUERY]", "Spyse search by a query"
9
+ method_option :title, type: :string, desc: "title"
10
+ method_option :description, type: :string, desc: "description"
11
+ method_option :tags, type: :array, desc: "tags"
12
+ method_option :type, type: :string, desc: "type to search (ip or domain)", default: "doamin"
13
+ def spyse(query)
14
+ with_error_handling do
15
+ run_analyzer Analyzers::Spyse, query: query, options: options
16
+ end
17
+ end
18
+ end
19
+ end
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,21 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Mihari
4
+ module Commands
5
+ module SSHFingerprint
6
+ def self.included(thor)
7
+ thor.class_eval do
8
+ desc "ssh_fingerprint [FINGERPRINT]", "Cross search with search engines by an SSH fingerprint (e.g. dc:14:de:8e:d7:c1:15:43:23:82:25:81:d2:59:e8:c0)"
9
+ method_option :title, type: :string, desc: "title"
10
+ method_option :description, type: :string, desc: "description"
11
+ method_option :tags, type: :array, desc: "tags"
12
+ def ssh_fingerprint(fingerprint)
13
+ with_error_handling do
14
+ run_analyzer Analyzers::SSHFingerprint, query: fingerprint, options: options
15
+ end
16
+ end
17
+ end
18
+ end
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Mihari
4
+ module Commands
5
+ module Urlscan
6
+ def self.included(thor)
7
+ thor.class_eval do
8
+ desc "urlscan [QUERY]", "urlscan search by a given query"
9
+ method_option :title, type: :string, desc: "title"
10
+ method_option :description, type: :string, desc: "description"
11
+ method_option :tags, type: :array, desc: "tags"
12
+ method_option :target_type, type: :string, default: "url", desc: "target type to fetch from lookup results (target type should be 'url', 'domain' or 'ip')"
13
+ method_option :use_similarity, type: :boolean, default: false, desc: "use similarity API or not"
14
+ def urlscan(query)
15
+ with_error_handling do
16
+ run_analyzer Analyzers::Urlscan, query: query, options: options
17
+ end
18
+ end
19
+ end
20
+ end
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,21 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Mihari
4
+ module Commands
5
+ module VirusTotal
6
+ def self.included(thor)
7
+ thor.class_eval do
8
+ desc "virustotal [IP|DOMAIN]", "VirusTotal resolutions lookup by an ip or domain"
9
+ method_option :title, type: :string, desc: "title"
10
+ method_option :description, type: :string, desc: "description"
11
+ method_option :tags, type: :array, desc: "tags"
12
+ def virustotal(indiactor)
13
+ with_error_handling do
14
+ run_analyzer Analyzers::VirusTotal, query: refang(indiactor), options: options
15
+ end
16
+ end
17
+ end
18
+ end
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Mihari
4
+ module Commands
5
+ module Web
6
+ def self.included(thor)
7
+ thor.class_eval do
8
+ desc "web", "Launch the web app"
9
+ method_option :port, type: :numeric, default: 9292
10
+ method_option :host, type: :string, default: "localhost"
11
+ def web
12
+ port = options["port"].to_i || 9292
13
+ host = options["host"] || "localhost"
14
+
15
+ load_configuration
16
+ Mihari::App.run!(port: port, host: host)
17
+ end
18
+ end
19
+ end
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Mihari
4
+ module Commands
5
+ module ZoomEye
6
+ def self.included(thor)
7
+ thor.class_eval do
8
+ desc "zoomeye [QUERY]", "ZoomEye search by a query"
9
+ method_option :title, type: :string, desc: "title"
10
+ method_option :description, type: :string, desc: "description"
11
+ method_option :tags, type: :array, desc: "tags"
12
+ method_option :type, type: :string, desc: "type to search(host / web)", default: "host"
13
+ def zoomeye(query)
14
+ with_error_handling do
15
+ run_analyzer Analyzers::ZoomEye, query: query, options: options
16
+ end
17
+ end
18
+ end
19
+ end
20
+ end
21
+ end
22
+ end
data/lib/mihari/config.rb CHANGED
@@ -4,7 +4,7 @@ require "yaml"
4
4
 
5
5
  module Mihari
6
6
  class Config
7
- attr_accessor :binaryedge_api_key, :censys_id, :censys_secret, :circl_passive_password, :circl_passive_username, :misp_api_endpoint, :misp_api_key, :onyphe_api_key, :otx_api_key, :passivetotal_api_key, :passivetotal_username, :pulsedive_api_key, :securitytrails_api_key, :shodan_api_key, :slack_channel, :slack_webhook_url, :spyse_api_key, :thehive_api_endpoint, :thehive_api_key, :urlscan_api_key, :virustotal_api_key, :zoomeye_password, :zoomeye_username, :database
7
+ attr_accessor :binaryedge_api_key, :censys_id, :censys_secret, :circl_passive_password, :circl_passive_username, :misp_api_endpoint, :misp_api_key, :onyphe_api_key, :otx_api_key, :passivetotal_api_key, :passivetotal_username, :pulsedive_api_key, :securitytrails_api_key, :shodan_api_key, :slack_channel, :slack_webhook_url, :spyse_api_key, :thehive_api_endpoint, :thehive_api_key, :urlscan_api_key, :virustotal_api_key, :zoomeye_api_key, :database
8
8
 
9
9
  def initialize
10
10
  load_from_env
@@ -32,8 +32,7 @@ module Mihari
32
32
  @thehive_api_key = ENV["THEHIVE_API_KEY"]
33
33
  @urlscan_api_key = ENV["URLSCAN_API_KEY"]
34
34
  @virustotal_api_key = ENV["VIRUSTOTAL_API_KEY"]
35
- @zoomeye_password = ENV["ZOOMEYE_PASSWORD"]
36
- @zoomeye_username = ENV["ZOOMEYE_USERNAME"]
35
+ @zoomeye_api_key = ENV["ZOOMEYE_API_KEY"]
37
36
 
38
37
  @database = ENV["DATABASE"] || "mihari.db"
39
38
  end
@@ -55,6 +54,18 @@ module Mihari
55
54
  end
56
55
  end
57
56
  end
57
+
58
+ def initialize_yaml(filename)
59
+ keys = new.instance_variables.map do |key|
60
+ key.to_s[1..]
61
+ end
62
+
63
+ config = keys.map do |key|
64
+ [key, nil]
65
+ end.to_h
66
+
67
+ YAML.dump(config, File.open(filename, "w"))
68
+ end
58
69
  end
59
70
  end
60
71
 
@@ -10,7 +10,7 @@ module Mihari
10
10
  return nil if config_keys.empty?
11
11
 
12
12
  config_keys.map do |key|
13
- {key: key.upcase, value: Mihari.config.send(key)}
13
+ { key: key.upcase, value: Mihari.config.send(key) }
14
14
  end
15
15
  end
16
16
 
@@ -25,25 +25,25 @@ module Mihari
25
25
  def vt_link
26
26
  return nil unless _vt_link
27
27
 
28
- {type: "button", text: "VirusTotal", url: _vt_link}
28
+ { type: "button", text: "VirusTotal", url: _vt_link }
29
29
  end
30
30
 
31
31
  def urlscan_link
32
32
  return nil unless _urlscan_link
33
33
 
34
- {type: "button", text: "urlscan.io", url: _urlscan_link}
34
+ { type: "button", text: "urlscan.io", url: _urlscan_link }
35
35
  end
36
36
 
37
37
  def censys_link
38
38
  return nil unless _censys_link
39
39
 
40
- {type: "button", text: "Censys", url: _censys_link}
40
+ { type: "button", text: "Censys", url: _censys_link }
41
41
  end
42
42
 
43
43
  def shodan_link
44
44
  return nil unless _shodan_link
45
45
 
46
- {type: "button", text: "Shodan", url: _shodan_link}
46
+ { type: "button", text: "Shodan", url: _shodan_link }
47
47
  end
48
48
 
49
49
  # @return [Array]
@@ -17,7 +17,7 @@ module Mihari
17
17
  api.alert.create(
18
18
  title: title,
19
19
  description: description,
20
- artifacts: artifacts.map { |artifact| {data: artifact.data, data_type: artifact.data_type, message: description} },
20
+ artifacts: artifacts.map { |artifact| { data: artifact.data, data_type: artifact.data_type, message: description } },
21
21
  tags: tags,
22
22
  type: "external",
23
23
  source: "mihari"
@@ -44,16 +44,16 @@ module Mihari
44
44
  relation = joins(:tags) if tag_name
45
45
  relation = joins(:artifacts) if artifact_data
46
46
 
47
- relation = relation.where(artifacts: {data: artifact_data}) if artifact_data
48
- relation = relation.where(tags: {name: tag_name}) if tag_name
47
+ relation = relation.where(artifacts: { data: artifact_data }) if artifact_data
48
+ relation = relation.where(tags: { name: tag_name }) if tag_name
49
49
 
50
50
  relation = relation.where(source: source) if source
51
51
  relation = relation.where(title: title) if title
52
52
 
53
- relation = relation.filter(description: {like: "%#{description}%"}) if description
53
+ relation = relation.filter(description: { like: "%#{description}%" }) if description
54
54
 
55
- relation = relation.filter(created_at: {gte: from_at}) if from_at
56
- relation = relation.filter(created_at: {lte: to_at}) if to_at
55
+ relation = relation.filter(created_at: { gte: from_at }) if from_at
56
+ relation = relation.filter(created_at: { lte: to_at }) if to_at
57
57
 
58
58
  relation
59
59
  end
@@ -1,6 +1,9 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require "active_record"
4
+ require "active_record/filter"
5
+ require "active_support/core_ext/integer/time"
6
+ require "active_support/core_ext/numeric/time"
4
7
 
5
8
  class ArtifactValidator < ActiveModel::Validator
6
9
  def validate(record)
@@ -20,8 +23,16 @@ module Mihari
20
23
  self.data_type = TypeChecker.type(data)
21
24
  end
22
25
 
23
- def unique?
24
- self.class.find_by(data: data).nil?
26
+ def unique?(ignore_old_artifacts: false, ignore_threshold: 0)
27
+ artifact = self.class.where(data: data).order(created_at: :desc).first
28
+ return true if artifact.nil?
29
+
30
+ return false unless ignore_old_artifacts
31
+
32
+ days_before = (-ignore_threshold).days.from_now
33
+ # if an artifact is created before {ignore_threshold} days, ignore it
34
+ # within {ignore_threshold} days, do not ignore it
35
+ artifact.created_at < days_before
25
36
  end
26
37
  end
27
38
  end
@@ -51,20 +51,20 @@ module Mihari
51
51
 
52
52
  def to_fields(clean_message, backtrace)
53
53
  fields = [
54
- {title: "Exception", value: clean_message},
55
- {title: "Hostname", value: hostname}
54
+ { title: "Exception", value: clean_message },
55
+ { title: "Hostname", value: hostname }
56
56
  ]
57
57
 
58
58
  if backtrace
59
59
  formatted_backtrace = format_backtrace(backtrace)
60
- fields << {title: "Backtrace", value: formatted_backtrace}
60
+ fields << { title: "Backtrace", value: formatted_backtrace }
61
61
  end
62
62
  fields
63
63
  end
64
64
 
65
65
  def hostname
66
66
  Socket.gethostname
67
- rescue => _e
67
+ rescue StandardError => _e
68
68
  "N/A"
69
69
  end
70
70
 
data/lib/mihari/status.rb CHANGED
@@ -3,7 +3,7 @@
3
3
  module Mihari
4
4
  class Status
5
5
  def check
6
- statuses.transform_values { |value| convert(**value) }
6
+ statuses
7
7
  end
8
8
 
9
9
  def self.check
@@ -12,14 +12,6 @@ module Mihari
12
12
 
13
13
  private
14
14
 
15
- def convert(is_configured:, values:, type:)
16
- {
17
- is_configured: is_configured,
18
- values: values,
19
- type: type
20
- }
21
- end
22
-
23
15
  def statuses
24
16
  (Mihari.analyzers + Mihari.emitters).map do |klass|
25
17
  name = klass.to_s.split("::").last.to_s
@@ -36,7 +28,7 @@ module Mihari
36
28
  values = instance.configuration_values
37
29
  type = is_analyzer ? "Analyzer" : "Emitter"
38
30
 
39
- values ? {is_configured: is_configured, values: values, type: type} : nil
31
+ values ? { is_configured: is_configured, values: values, type: type } : nil
40
32
  rescue ArgumentError => _e
41
33
  nil
42
34
  end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Mihari
4
- VERSION = "2.0.0"
4
+ VERSION = "2.3.1"
5
5
  end
@@ -1,119 +1,44 @@
1
- require "awrence"
1
+ # frozen_string_literal: true
2
+
2
3
  require "launchy"
3
4
  require "rack"
5
+ require "rack/handler/puma"
4
6
  require "sinatra"
5
- require "sinatra/json"
6
- require "sinatra/reloader"
7
+
8
+ require "mihari/web/helpers/json"
9
+
10
+ require "mihari/web/controllers/base_controller"
11
+
12
+ require "mihari/web/controllers/alerts_controller"
13
+ require "mihari/web/controllers/artifacts_controller"
14
+ require "mihari/web/controllers/command_controller"
15
+ require "mihari/web/controllers/config_controller"
16
+ require "mihari/web/controllers/sources_controller"
17
+ require "mihari/web/controllers/tags_controller"
7
18
 
8
19
  module Mihari
9
20
  class App < Sinatra::Base
10
- register Sinatra::Reloader
11
-
12
21
  set :root, File.dirname(__FILE__)
13
- set :public_folder, proc { File.join(root, "public") }
22
+ set :public_folder, File.join(root, "public")
14
23
 
15
24
  get "/" do
16
25
  send_file File.join(settings.public_folder, "index.html")
17
26
  end
18
27
 
19
- get "/api/alerts" do
20
- page = params["page"] || 1
21
- page = page.to_i
22
- limit = 10
23
-
24
- artifact_data = params["artifact"]
25
- description = params["description"]
26
- source = params["source"]
27
- tag_name = params["tag"]
28
- title = params["title"]
29
-
30
- from_at = params["from_at"] || params["fromAt"]
31
- from_at = DateTime.parse(from_at) if from_at
32
- to_at = params["to_at"] || params["toAt"]
33
- to_at = DateTime.parse(to_at) if to_at
34
-
35
- alerts = Mihari::Alert.search(
36
- artifact_data: artifact_data,
37
- description: description,
38
- from_at: from_at,
39
- limit: limit,
40
- page: page,
41
- source: source,
42
- tag_name: tag_name,
43
- title: title,
44
- to_at: to_at
45
- )
46
- total = Mihari::Alert.count(
47
- artifact_data: artifact_data,
48
- description: description,
49
- from_at: from_at,
50
- source: source,
51
- tag_name: tag_name,
52
- title: title,
53
- to_at: to_at
54
- )
55
-
56
- json = {alerts: alerts, total: total, current_page: page, page_size: limit}
57
- json json.to_camelback_keys
58
- end
59
-
60
- delete "/api/alerts/:id" do
61
- id = params["id"]
62
- id = id.to_i
63
-
64
- begin
65
- alert = Mihari::Alert.find(id)
66
- alert.destroy
67
-
68
- status 204
69
- body ""
70
- rescue ActiveRecord::RecordNotFound
71
- status 404
72
-
73
- message = {message: "ID:#{id} is not found"}
74
- json message
75
- end
76
- end
77
-
78
- delete "/api/artifacts/:id" do
79
- id = params["id"]
80
- id = id.to_i
81
-
82
- begin
83
- alert = Mihari::Artifact.find(id)
84
- alert.delete
85
-
86
- status 204
87
- body ""
88
- rescue ActiveRecord::RecordNotFound
89
- status 404
90
-
91
- message = {message: "ID:#{id} is not found"}
92
- json message
93
- end
94
- end
95
-
96
- get "/api/sources" do
97
- tags = Mihari::Alert.distinct.pluck(:source)
98
- json tags
99
- end
100
-
101
- get "/api/tags" do
102
- tags = Mihari::Tag.distinct.pluck(:name)
103
- json tags
104
- end
105
-
106
- get "/api/config" do
107
- report = Status.check
108
-
109
- json report.to_camelback_keys
110
- end
28
+ use Mihari::Controllers::AlertsController
29
+ use Mihari::Controllers::ArtifactsController
30
+ use Mihari::Controllers::CommandController
31
+ use Mihari::Controllers::ConfigController
32
+ use Mihari::Controllers::SourcesController
33
+ use Mihari::Controllers::TagsController
111
34
 
112
35
  class << self
113
36
  def run!(port: 9292, host: "localhost")
114
- Launchy.open "http://#{host}:#{port}"
37
+ url = "http://#{host}:#{port}"
38
+
39
+ Rack::Handler::Puma.run self, Port: port, Host: host do |server|
40
+ Launchy.open url
115
41
 
116
- Rack::Handler::WEBrick.run self, Port: port, Host: host do |server|
117
42
  [:INT, :TERM].each do |sig|
118
43
  trap(sig) do
119
44
  server.shutdown