mihari 1.3.1 → 1.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/test.yml +44 -0
  3. data/README.md +23 -12
  4. data/Rakefile +1 -0
  5. data/docker/Dockerfile +3 -2
  6. data/{screenshots → images}/alert.png +0 -0
  7. data/{screenshots → images}/eyecatch.png +0 -0
  8. data/images/logo.png +0 -0
  9. data/{screenshots → images}/misp.png +0 -0
  10. data/{screenshots → images}/slack.png +0 -0
  11. data/lib/mihari/alert_viewer.rb +3 -3
  12. data/lib/mihari/analyzers/base.rb +1 -1
  13. data/lib/mihari/analyzers/basic.rb +3 -4
  14. data/lib/mihari/analyzers/binaryedge.rb +4 -7
  15. data/lib/mihari/analyzers/censys.rb +3 -7
  16. data/lib/mihari/analyzers/circl.rb +3 -5
  17. data/lib/mihari/analyzers/crtsh.rb +2 -6
  18. data/lib/mihari/analyzers/dnpedia.rb +3 -6
  19. data/lib/mihari/analyzers/dnstwister.rb +4 -9
  20. data/lib/mihari/analyzers/free_text.rb +2 -6
  21. data/lib/mihari/analyzers/http_hash.rb +3 -11
  22. data/lib/mihari/analyzers/onyphe.rb +3 -6
  23. data/lib/mihari/analyzers/otx.rb +4 -9
  24. data/lib/mihari/analyzers/passive_dns.rb +4 -9
  25. data/lib/mihari/analyzers/passive_ssl.rb +4 -9
  26. data/lib/mihari/analyzers/passivetotal.rb +9 -14
  27. data/lib/mihari/analyzers/pulsedive.rb +7 -12
  28. data/lib/mihari/analyzers/reverse_whois.rb +4 -9
  29. data/lib/mihari/analyzers/securitytrails.rb +12 -17
  30. data/lib/mihari/analyzers/securitytrails_domain_feed.rb +3 -7
  31. data/lib/mihari/analyzers/shodan.rb +9 -8
  32. data/lib/mihari/analyzers/spyse.rb +6 -11
  33. data/lib/mihari/analyzers/ssh_fingerprint.rb +2 -6
  34. data/lib/mihari/analyzers/urlscan.rb +21 -9
  35. data/lib/mihari/analyzers/virustotal.rb +6 -11
  36. data/lib/mihari/analyzers/zoomeye.rb +7 -11
  37. data/lib/mihari/cli.rb +14 -7
  38. data/lib/mihari/config.rb +1 -25
  39. data/lib/mihari/database.rb +1 -1
  40. data/lib/mihari/emitters/misp.rb +4 -2
  41. data/lib/mihari/emitters/slack.rb +18 -7
  42. data/lib/mihari/emitters/the_hive.rb +2 -2
  43. data/lib/mihari/errors.rb +2 -0
  44. data/lib/mihari/models/artifact.rb +1 -1
  45. data/lib/mihari/notifiers/exception_notifier.rb +5 -5
  46. data/lib/mihari/status.rb +1 -1
  47. data/lib/mihari/type_checker.rb +4 -4
  48. data/lib/mihari/version.rb +1 -1
  49. data/mihari.gemspec +23 -24
  50. metadata +44 -57
  51. data/.travis.yml +0 -13
@@ -5,11 +5,7 @@ require "parallel"
5
5
  module Mihari
6
6
  module Analyzers
7
7
  class SSHFingerprint < Base
8
- attr_reader :fingerprint
9
-
10
- attr_reader :title
11
- attr_reader :description
12
- attr_reader :tags
8
+ attr_reader :fingerprint, :title, :description, :tags
13
9
 
14
10
  def initialize(fingerprint, title: nil, description: nil, tags: [])
15
11
  super()
@@ -46,7 +42,7 @@ module Mihari
46
42
 
47
43
  [
48
44
  binary_edge,
49
- shodan,
45
+ shodan
50
46
  ].compact
51
47
  end
52
48
 
@@ -5,20 +5,29 @@ require "urlscan"
5
5
  module Mihari
6
6
  module Analyzers
7
7
  class Urlscan < Base
8
- attr_reader :title
9
- attr_reader :description
10
- attr_reader :query
11
- attr_reader :tags
12
- attr_reader :target_type
8
+ attr_reader :title, :description, :query, :tags, :filter, :target_type, :use_pro, :use_similarity
13
9
 
14
- def initialize(query, title: nil, description: nil, tags: [], target_type: "url")
10
+ def initialize(
11
+ query,
12
+ description: nil,
13
+ filter: nil,
14
+ tags: [],
15
+ target_type: "url",
16
+ title: nil,
17
+ use_pro: false,
18
+ use_similarity: false
19
+ )
15
20
  super()
16
21
 
17
22
  @query = query
18
23
  @title = title || "urlscan lookup"
19
24
  @description = description || "query = #{query}"
20
25
  @tags = tags
26
+
27
+ @filter = filter
21
28
  @target_type = target_type
29
+ @use_pro = use_pro
30
+ @use_similarity = use_similarity
22
31
 
23
32
  raise InvalidInputError, "type should be url, domain or ip." unless valid_target_type?
24
33
  end
@@ -27,7 +36,7 @@ module Mihari
27
36
  result = search
28
37
  return [] unless result
29
38
 
30
- results = result.dig("results") || []
39
+ results = result["results"] || []
31
40
  results.map do |match|
32
41
  match.dig "page", target_type
33
42
  end.compact.uniq
@@ -36,7 +45,7 @@ module Mihari
36
45
  private
37
46
 
38
47
  def config_keys
39
- %w(urlscan_api_key)
48
+ %w[urlscan_api_key]
40
49
  end
41
50
 
42
51
  def api
@@ -44,11 +53,14 @@ module Mihari
44
53
  end
45
54
 
46
55
  def search
56
+ return api.pro.similar(query) if use_similarity
57
+ return api.pro.search(query: query, filter: filter, size: 10_000) if use_pro
58
+
47
59
  api.search(query, size: 10_000)
48
60
  end
49
61
 
50
62
  def valid_target_type?
51
- %w(url domain ip).include? target_type
63
+ %w[url domain ip].include? target_type
52
64
  end
53
65
  end
54
66
  end
@@ -5,12 +5,7 @@ require "virustotal"
5
5
  module Mihari
6
6
  module Analyzers
7
7
  class VirusTotal < Base
8
- attr_reader :indicator
9
- attr_reader :type
10
-
11
- attr_reader :title
12
- attr_reader :description
13
- attr_reader :tags
8
+ attr_reader :indicator, :type, :title, :description, :tags
14
9
 
15
10
  def initialize(indicator, title: nil, description: nil, tags: [])
16
11
  super()
@@ -30,7 +25,7 @@ module Mihari
30
25
  private
31
26
 
32
27
  def config_keys
33
- %w(virustotal_api_key)
28
+ %w[virustotal_api_key]
34
29
  end
35
30
 
36
31
  def api
@@ -38,7 +33,7 @@ module Mihari
38
33
  end
39
34
 
40
35
  def valid_type?
41
- %w(ip domain).include? type
36
+ %w[ip domain].include? type
42
37
  end
43
38
 
44
39
  def lookup
@@ -48,14 +43,14 @@ module Mihari
48
43
  when "ip"
49
44
  ip_lookup
50
45
  else
51
- raise InvalidInputError, "#{indicator}(type: #{type || 'unknown'}) is not supported." unless valid_type?
46
+ raise InvalidInputError, "#{indicator}(type: #{type || "unknown"}) is not supported." unless valid_type?
52
47
  end
53
48
  end
54
49
 
55
50
  def domain_lookup
56
51
  res = api.domain.resolutions(indicator)
57
52
 
58
- data = res.dig("data") || []
53
+ data = res["data"] || []
59
54
  data.map do |item|
60
55
  item.dig("attributes", "ip_address")
61
56
  end.compact.uniq
@@ -64,7 +59,7 @@ module Mihari
64
59
  def ip_lookup
65
60
  res = api.ip_address.resolutions(indicator)
66
61
 
67
- data = res.dig("data") || []
62
+ data = res["data"] || []
68
63
  data.map do |item|
69
64
  item.dig("attributes", "host_name")
70
65
  end.compact.uniq
@@ -5,11 +5,7 @@ require "zoomeye"
5
5
  module Mihari
6
6
  module Analyzers
7
7
  class ZoomEye < Base
8
- attr_reader :title
9
- attr_reader :description
10
- attr_reader :query
11
- attr_reader :tags
12
- attr_reader :type
8
+ attr_reader :title, :description, :query, :tags, :type
13
9
 
14
10
  def initialize(query, title: nil, description: nil, tags: [], type: "host")
15
11
  super()
@@ -37,11 +33,11 @@ module Mihari
37
33
  PAGE_SIZE = 10
38
34
 
39
35
  def valid_type?
40
- %w(host web).include? type
36
+ %w[host web].include? type
41
37
  end
42
38
 
43
39
  def config_keys
44
- %w(zoomeye_password zoomeye_username)
40
+ %w[zoomeye_password zoomeye_username]
45
41
  end
46
42
 
47
43
  def api
@@ -50,9 +46,9 @@ module Mihari
50
46
 
51
47
  def convert_responses(responses)
52
48
  responses.map do |res|
53
- matches = res.dig("matches") || []
49
+ matches = res["matches"] || []
54
50
  matches.map do |match|
55
- match.dig "ip"
51
+ match["ip"]
56
52
  end
57
53
  end.flatten.compact.uniq
58
54
  end
@@ -69,7 +65,7 @@ module Mihari
69
65
  res = _host_lookup(query, page: page)
70
66
  break unless res
71
67
 
72
- total = res.dig("total").to_i
68
+ total = res["total"].to_i
73
69
  responses << res
74
70
  break if total <= page * PAGE_SIZE
75
71
  end
@@ -88,7 +84,7 @@ module Mihari
88
84
  res = _web_lookup(query, page: page)
89
85
  break unless res
90
86
 
91
- total = res.dig("total").to_i
87
+ total = res["total"].to_i
92
88
  responses << res
93
89
  break if total <= page * PAGE_SIZE
94
90
  end
data/lib/mihari/cli.rb CHANGED
@@ -7,6 +7,10 @@ module Mihari
7
7
  class CLI < Thor
8
8
  class_option :config, type: :string, desc: "path to config file"
9
9
 
10
+ def self.exit_on_failure?
11
+ true
12
+ end
13
+
10
14
  desc "censys [QUERY]", "Censys IPv4 search by a query"
11
15
  method_option :title, type: :string, desc: "title"
12
16
  method_option :description, type: :string, desc: "description"
@@ -42,7 +46,10 @@ module Mihari
42
46
  method_option :title, type: :string, desc: "title"
43
47
  method_option :description, type: :string, desc: "description"
44
48
  method_option :tags, type: :array, desc: "tags"
49
+ method_option :filter, type: :string, desc: "filter for urlscan pro search"
45
50
  method_option :target_type, type: :string, default: "url", desc: "target type to fetch from lookup results (target type should be 'url', 'domain' or 'ip')"
51
+ method_option :use_pro, type: :boolean, default: false, desc: "use pro search API or not"
52
+ method_option :use_similarity, type: :boolean, default: false, desc: "use similarity API or not"
46
53
  def urlscan(query)
47
54
  with_error_handling do
48
55
  run_analyzer Analyzers::Urlscan, query: query, options: options
@@ -252,16 +259,16 @@ module Mihari
252
259
  desc "import_from_json", "Give a JSON input via STDIN"
253
260
  def import_from_json(input = nil)
254
261
  with_error_handling do
255
- json = input || STDIN.gets.chomp
262
+ json = input || $stdin.gets.chomp
256
263
  raise ArgumentError, "Input not found: please give an input in a JSON format" unless json
257
264
 
258
265
  json = parse_as_json(json)
259
266
  raise ArgumentError, "Invalid input format: an input JSON data should have title, description and artifacts key" unless valid_json?(json)
260
267
 
261
- title = json.dig("title")
262
- description = json.dig("description")
263
- artifacts = json.dig("artifacts")
264
- tags = json.dig("tags") || []
268
+ title = json["title"]
269
+ description = json["description"]
270
+ artifacts = json["artifacts"]
271
+ tags = json["tags"] || []
265
272
 
266
273
  basic = Analyzers::Basic.new(title: title, description: description, artifacts: artifacts, source: "json", tags: tags)
267
274
  basic.run
@@ -295,7 +302,7 @@ module Mihari
295
302
  no_commands do
296
303
  def with_error_handling
297
304
  yield
298
- rescue StandardError => e
305
+ rescue => e
299
306
  notifier = Notifiers::ExceptionNotifier.new
300
307
  notifier.notify e
301
308
  end
@@ -308,7 +315,7 @@ module Mihari
308
315
 
309
316
  # @return [true, false]
310
317
  def valid_json?(json)
311
- %w(title description artifacts).all? { |key| json.key? key }
318
+ %w[title description artifacts].all? { |key| json.key? key }
312
319
  end
313
320
 
314
321
  def load_configuration
data/lib/mihari/config.rb CHANGED
@@ -4,31 +4,7 @@ require "yaml"
4
4
 
5
5
  module Mihari
6
6
  class Config
7
- attr_accessor :binaryedge_api_key
8
- attr_accessor :censys_id
9
- attr_accessor :censys_secret
10
- attr_accessor :circl_passive_password
11
- attr_accessor :circl_passive_username
12
- attr_accessor :misp_api_endpoint
13
- attr_accessor :misp_api_key
14
- attr_accessor :onyphe_api_key
15
- attr_accessor :otx_api_key
16
- attr_accessor :passivetotal_api_key
17
- attr_accessor :passivetotal_username
18
- attr_accessor :pulsedive_api_key
19
- attr_accessor :securitytrails_api_key
20
- attr_accessor :shodan_api_key
21
- attr_accessor :slack_channel
22
- attr_accessor :slack_webhook_url
23
- attr_accessor :spyse_api_key
24
- attr_accessor :thehive_api_endpoint
25
- attr_accessor :thehive_api_key
26
- attr_accessor :urlscan_api_key
27
- attr_accessor :virustotal_api_key
28
- attr_accessor :zoomeye_password
29
- attr_accessor :zoomeye_username
30
-
31
- attr_accessor :database
7
+ attr_accessor :binaryedge_api_key, :censys_id, :censys_secret, :circl_passive_password, :circl_passive_username, :misp_api_endpoint, :misp_api_key, :onyphe_api_key, :otx_api_key, :passivetotal_api_key, :passivetotal_username, :pulsedive_api_key, :securitytrails_api_key, :shodan_api_key, :slack_channel, :slack_webhook_url, :spyse_api_key, :thehive_api_endpoint, :thehive_api_key, :urlscan_api_key, :virustotal_api_key, :zoomeye_password, :zoomeye_username, :database
32
8
 
33
9
  def initialize
34
10
  load_from_env
@@ -54,7 +54,7 @@ module Mihari
54
54
 
55
55
  ActiveRecord::Migration.verbose = false
56
56
  InitialSchema.migrate(:up)
57
- rescue StandardError
57
+ rescue
58
58
  # Do nothing
59
59
  end
60
60
 
@@ -7,6 +7,8 @@ module Mihari
7
7
  module Emitters
8
8
  class MISP < Base
9
9
  def initialize
10
+ super()
11
+
10
12
  ::MISP.configure do |config|
11
13
  config.api_endpoint = Mihari.config.misp_api_endpoint
12
14
  config.api_key = Mihari.config.misp_api_key
@@ -35,7 +37,7 @@ module Mihari
35
37
  private
36
38
 
37
39
  def config_keys
38
- %w(misp_api_endpoint misp_api_key)
40
+ %w[misp_api_endpoint misp_api_key]
39
41
  end
40
42
 
41
43
  def build_attribute(artifact)
@@ -61,7 +63,7 @@ module Mihari
61
63
  ip: "ip-dst",
62
64
  mail: "email-dst",
63
65
  url: "url",
64
- domain: "domain",
66
+ domain: "domain"
65
67
  }
66
68
  return table[type] if table.key?(type)
67
69
 
@@ -19,25 +19,31 @@ module Mihari
19
19
  end
20
20
 
21
21
  def actions
22
- [vt_link, urlscan_link, censys_link].compact
22
+ [vt_link, urlscan_link, censys_link, shodan_link].compact
23
23
  end
24
24
 
25
25
  def vt_link
26
26
  return nil unless _vt_link
27
27
 
28
- { type: "button", text: "Lookup on VirusTotal", url: _vt_link, }
28
+ { type: "button", text: "VirusTotal", url: _vt_link }
29
29
  end
30
30
 
31
31
  def urlscan_link
32
32
  return nil unless _urlscan_link
33
33
 
34
- { type: "button", text: "Lookup on urlscan.io", url: _urlscan_link, }
34
+ { type: "button", text: "urlscan.io", url: _urlscan_link }
35
35
  end
36
36
 
37
37
  def censys_link
38
38
  return nil unless _censys_link
39
39
 
40
- { type: "button", text: "Lookup on Censys", url: _censys_link, }
40
+ { type: "button", text: "Censys", url: _censys_link }
41
+ end
42
+
43
+ def shodan_link
44
+ return nil unless _shodan_link
45
+
46
+ { type: "button", text: "Shodan", url: _shodan_link }
41
47
  end
42
48
 
43
49
  # @return [Array]
@@ -89,6 +95,11 @@ module Mihari
89
95
  end
90
96
  memoize :_censys_link
91
97
 
98
+ def _shodan_link
99
+ data_type == "ip" ? "https://www.shodan.io/host/#{data}" : nil
100
+ end
101
+ memoize :_shodan_link
102
+
92
103
  # @return [String]
93
104
  def sha256
94
105
  Digest::SHA256.hexdigest data
@@ -96,7 +107,7 @@ module Mihari
96
107
 
97
108
  # @return [String]
98
109
  def defanged_data
99
- @defanged_data ||= data.to_s.gsub /\./, "[.]"
110
+ @defanged_data ||= data.to_s.gsub(/\./, "[.]")
100
111
  end
101
112
  end
102
113
 
@@ -121,7 +132,7 @@ module Mihari
121
132
  [
122
133
  "*#{title}*",
123
134
  "*Desc.*: #{description}",
124
- "*Tags*: #{tags.join(', ')}",
135
+ "*Tags*: #{tags.join(', ')}"
125
136
  ].join("\n")
126
137
  end
127
138
 
@@ -137,7 +148,7 @@ module Mihari
137
148
  private
138
149
 
139
150
  def config_keys
140
- %w(slack_webhook_url)
151
+ %w[slack_webhook_url]
141
152
  end
142
153
  end
143
154
  end
@@ -17,7 +17,7 @@ module Mihari
17
17
  api.alert.create(
18
18
  title: title,
19
19
  description: description,
20
- artifacts: artifacts.map { |artifact| { data: artifact.data, data_type: artifact.data_type, message: description } },
20
+ artifacts: artifacts.map { |artifact| {data: artifact.data, data_type: artifact.data_type, message: description} },
21
21
  tags: tags,
22
22
  type: "external",
23
23
  source: "mihari"
@@ -27,7 +27,7 @@ module Mihari
27
27
  private
28
28
 
29
29
  def config_keys
30
- %w(thehive_api_endpoint thehive_api_key)
30
+ %w[thehive_api_endpoint thehive_api_key]
31
31
  end
32
32
 
33
33
  def api
data/lib/mihari/errors.rb CHANGED
@@ -2,6 +2,8 @@
2
2
 
3
3
  module Mihari
4
4
  class Error < StandardError; end
5
+
5
6
  class InvalidInputError < Error; end
7
+
6
8
  class RetryableError < Error; end
7
9
  end
@@ -6,7 +6,7 @@ class ArtifactValidator < ActiveModel::Validator
6
6
  def validate(record)
7
7
  return if record.data_type
8
8
 
9
- record.errors[:data] << "#{record.data} is not supported"
9
+ record.errors.add :data, "#{record.data} is not supported"
10
10
  end
11
11
  end
12
12