mihari 1.3.1 → 1.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/test.yml +44 -0
  3. data/README.md +23 -12
  4. data/Rakefile +1 -0
  5. data/docker/Dockerfile +3 -2
  6. data/{screenshots → images}/alert.png +0 -0
  7. data/{screenshots → images}/eyecatch.png +0 -0
  8. data/images/logo.png +0 -0
  9. data/{screenshots → images}/misp.png +0 -0
  10. data/{screenshots → images}/slack.png +0 -0
  11. data/lib/mihari/alert_viewer.rb +3 -3
  12. data/lib/mihari/analyzers/base.rb +1 -1
  13. data/lib/mihari/analyzers/basic.rb +3 -4
  14. data/lib/mihari/analyzers/binaryedge.rb +4 -7
  15. data/lib/mihari/analyzers/censys.rb +3 -7
  16. data/lib/mihari/analyzers/circl.rb +3 -5
  17. data/lib/mihari/analyzers/crtsh.rb +2 -6
  18. data/lib/mihari/analyzers/dnpedia.rb +3 -6
  19. data/lib/mihari/analyzers/dnstwister.rb +4 -9
  20. data/lib/mihari/analyzers/free_text.rb +2 -6
  21. data/lib/mihari/analyzers/http_hash.rb +3 -11
  22. data/lib/mihari/analyzers/onyphe.rb +3 -6
  23. data/lib/mihari/analyzers/otx.rb +4 -9
  24. data/lib/mihari/analyzers/passive_dns.rb +4 -9
  25. data/lib/mihari/analyzers/passive_ssl.rb +4 -9
  26. data/lib/mihari/analyzers/passivetotal.rb +9 -14
  27. data/lib/mihari/analyzers/pulsedive.rb +7 -12
  28. data/lib/mihari/analyzers/reverse_whois.rb +4 -9
  29. data/lib/mihari/analyzers/securitytrails.rb +12 -17
  30. data/lib/mihari/analyzers/securitytrails_domain_feed.rb +3 -7
  31. data/lib/mihari/analyzers/shodan.rb +9 -8
  32. data/lib/mihari/analyzers/spyse.rb +6 -11
  33. data/lib/mihari/analyzers/ssh_fingerprint.rb +2 -6
  34. data/lib/mihari/analyzers/urlscan.rb +21 -9
  35. data/lib/mihari/analyzers/virustotal.rb +6 -11
  36. data/lib/mihari/analyzers/zoomeye.rb +7 -11
  37. data/lib/mihari/cli.rb +14 -7
  38. data/lib/mihari/config.rb +1 -25
  39. data/lib/mihari/database.rb +1 -1
  40. data/lib/mihari/emitters/misp.rb +4 -2
  41. data/lib/mihari/emitters/slack.rb +18 -7
  42. data/lib/mihari/emitters/the_hive.rb +2 -2
  43. data/lib/mihari/errors.rb +2 -0
  44. data/lib/mihari/models/artifact.rb +1 -1
  45. data/lib/mihari/notifiers/exception_notifier.rb +5 -5
  46. data/lib/mihari/status.rb +1 -1
  47. data/lib/mihari/type_checker.rb +4 -4
  48. data/lib/mihari/version.rb +1 -1
  49. data/mihari.gemspec +23 -24
  50. metadata +44 -57
  51. data/.travis.yml +0 -13
@@ -5,10 +5,7 @@ require "onyphe"
5
5
  module Mihari
6
6
  module Analyzers
7
7
  class Onyphe < Base
8
- attr_reader :title
9
- attr_reader :description
10
- attr_reader :query
11
- attr_reader :tags
8
+ attr_reader :title, :description, :query, :tags
12
9
 
13
10
  def initialize(query, title: nil, description: nil, tags: [])
14
11
  super()
@@ -35,7 +32,7 @@ module Mihari
35
32
  PAGE_SIZE = 10
36
33
 
37
34
  def config_keys
38
- %w(onyphe_api_key)
35
+ %w[onyphe_api_key]
39
36
  end
40
37
 
41
38
  def api
@@ -51,7 +48,7 @@ module Mihari
51
48
  (1..Float::INFINITY).each do |page|
52
49
  res = search_with_page(query, page: page)
53
50
  responses << res
54
- total = res.dig("total").to_i
51
+ total = res["total"].to_i
55
52
  break if total <= page * PAGE_SIZE
56
53
  end
57
54
  responses
@@ -5,12 +5,7 @@ require "otx_ruby"
5
5
  module Mihari
6
6
  module Analyzers
7
7
  class OTX < Base
8
- attr_reader :query
9
- attr_reader :type
10
-
11
- attr_reader :title
12
- attr_reader :description
13
- attr_reader :tags
8
+ attr_reader :query, :type, :title, :description, :tags
14
9
 
15
10
  def initialize(query, title: nil, description: nil, tags: [])
16
11
  super()
@@ -30,7 +25,7 @@ module Mihari
30
25
  private
31
26
 
32
27
  def config_keys
33
- %w(otx_api_key)
28
+ %w[otx_api_key]
34
29
  end
35
30
 
36
31
  def domain_client
@@ -42,7 +37,7 @@ module Mihari
42
37
  end
43
38
 
44
39
  def valid_type?
45
- %w(ip domain).include? type
40
+ %w[ip domain].include? type
46
41
  end
47
42
 
48
43
  def lookup
@@ -52,7 +47,7 @@ module Mihari
52
47
  when "ip"
53
48
  ip_lookup
54
49
  else
55
- raise InvalidInputError, "#{query}(type: #{type || 'unknown'}) is not supported." unless valid_type?
50
+ raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
56
51
  end
57
52
  end
58
53
 
@@ -5,12 +5,7 @@ require "parallel"
5
5
  module Mihari
6
6
  module Analyzers
7
7
  class PassiveDNS < Base
8
- attr_reader :query
9
- attr_reader :type
10
-
11
- attr_reader :title
12
- attr_reader :description
13
- attr_reader :tags
8
+ attr_reader :query, :type, :title, :description, :tags
14
9
 
15
10
  ANALYZERS = [
16
11
  Mihari::Analyzers::CIRCL,
@@ -18,7 +13,7 @@ module Mihari
18
13
  Mihari::Analyzers::PassiveTotal,
19
14
  Mihari::Analyzers::Pulsedive,
20
15
  Mihari::Analyzers::SecurityTrails,
21
- Mihari::Analyzers::VirusTotal,
16
+ Mihari::Analyzers::VirusTotal
22
17
  ].freeze
23
18
 
24
19
  def initialize(query, title: nil, description: nil, tags: [])
@@ -41,11 +36,11 @@ module Mihari
41
36
  private
42
37
 
43
38
  def valid_type?
44
- %w(ip domain).include? type
39
+ %w[ip domain].include? type
45
40
  end
46
41
 
47
42
  def analyzers
48
- raise InvalidInputError, "#{query}(type: #{type || 'unknown'}) is not supported." unless valid_type?
43
+ raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
49
44
 
50
45
  ANALYZERS.map do |klass|
51
46
  klass.new(query)
@@ -5,16 +5,11 @@ require "parallel"
5
5
  module Mihari
6
6
  module Analyzers
7
7
  class PassiveSSL < Base
8
- attr_reader :query
9
- attr_reader :type
10
-
11
- attr_reader :title
12
- attr_reader :description
13
- attr_reader :tags
8
+ attr_reader :query, :type, :title, :description, :tags
14
9
 
15
10
  ANALYZERS = [
16
11
  Mihari::Analyzers::CIRCL,
17
- Mihari::Analyzers::PassiveTotal,
12
+ Mihari::Analyzers::PassiveTotal
18
13
  ].freeze
19
14
 
20
15
  def initialize(query, title: nil, description: nil, tags: [])
@@ -37,11 +32,11 @@ module Mihari
37
32
  private
38
33
 
39
34
  def valid_type?
40
- %w(sha1).include? type
35
+ %w[sha1].include? type
41
36
  end
42
37
 
43
38
  def analyzers
44
- raise InvalidInputError, "#{query}(type: #{type || 'unknown'}) is not supported." unless valid_type?
39
+ raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
45
40
 
46
41
  ANALYZERS.map do |klass|
47
42
  klass.new(query)
@@ -5,12 +5,7 @@ require "passivetotal"
5
5
  module Mihari
6
6
  module Analyzers
7
7
  class PassiveTotal < Base
8
- attr_reader :query
9
- attr_reader :type
10
-
11
- attr_reader :title
12
- attr_reader :description
13
- attr_reader :tags
8
+ attr_reader :query, :type, :title, :description, :tags
14
9
 
15
10
  def initialize(query, title: nil, description: nil, tags: [])
16
11
  super()
@@ -30,7 +25,7 @@ module Mihari
30
25
  private
31
26
 
32
27
  def config_keys
33
- %w(passivetotal_username passivetotal_api_key)
28
+ %w[passivetotal_username passivetotal_api_key]
34
29
  end
35
30
 
36
31
  def api
@@ -38,7 +33,7 @@ module Mihari
38
33
  end
39
34
 
40
35
  def valid_type?
41
- %w(ip domain mail).include? type
36
+ %w[ip domain mail].include? type
42
37
  end
43
38
 
44
39
  def lookup
@@ -52,28 +47,28 @@ module Mihari
52
47
  when "hash"
53
48
  ssl_lookup
54
49
  else
55
- raise InvalidInputError, "#{query}(type: #{type || 'unknown'}) is not supported." unless valid_type?
50
+ raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
56
51
  end
57
52
  end
58
53
 
59
54
  def passive_dns_lookup
60
55
  res = api.dns.passive_unique(query)
61
- res.dig("results") || []
56
+ res["results"] || []
62
57
  end
63
58
 
64
59
  def reverse_whois_lookup
65
60
  res = api.whois.search(query: query, field: "email")
66
- results = res.dig("results") || []
61
+ results = res["results"] || []
67
62
  results.map do |result|
68
- result.dig("domain")
63
+ result["domain"]
69
64
  end.flatten.compact.uniq
70
65
  end
71
66
 
72
67
  def ssl_lookup
73
68
  res = api.ssl.history(query)
74
- results = res.dig("results") || []
69
+ results = res["results"] || []
75
70
  results.map do |result|
76
- result.dig("ipAddresses")
71
+ result["ipAddresses"]
77
72
  end.flatten.compact.uniq
78
73
  end
79
74
  end
@@ -5,12 +5,7 @@ require "pulsedive"
5
5
  module Mihari
6
6
  module Analyzers
7
7
  class Pulsedive < Base
8
- attr_reader :query
9
- attr_reader :type
10
-
11
- attr_reader :title
12
- attr_reader :description
13
- attr_reader :tags
8
+ attr_reader :query, :type, :title, :description, :tags
14
9
 
15
10
  def initialize(query, title: nil, description: nil, tags: [])
16
11
  super()
@@ -30,7 +25,7 @@ module Mihari
30
25
  private
31
26
 
32
27
  def config_keys
33
- %w(pulsedive_api_key)
28
+ %w[pulsedive_api_key]
34
29
  end
35
30
 
36
31
  def api
@@ -38,18 +33,18 @@ module Mihari
38
33
  end
39
34
 
40
35
  def valid_type?
41
- %w(ip domain).include? type
36
+ %w[ip domain].include? type
42
37
  end
43
38
 
44
39
  def lookup
45
- raise InvalidInputError, "#{query}(type: #{type || 'unknown'}) is not supported." unless valid_type?
40
+ raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
46
41
 
47
42
  indicator = api.indicator.get_by_value(query)
48
- iid = indicator.dig("iid")
43
+ iid = indicator["iid"]
49
44
 
50
45
  properties = api.indicator.get_properties_by_id(iid)
51
- (properties.dig("dns") || []).map do |property|
52
- property.dig("value") if ["A", "PTR"].include?(property.dig("name"))
46
+ (properties["dns"] || []).map do |property|
47
+ property["value"] if ["A", "PTR"].include?(property["name"])
53
48
  end.compact
54
49
  end
55
50
  end
@@ -5,16 +5,11 @@ require "parallel"
5
5
  module Mihari
6
6
  module Analyzers
7
7
  class ReveseWhois < Base
8
- attr_reader :query
9
- attr_reader :type
10
-
11
- attr_reader :title
12
- attr_reader :description
13
- attr_reader :tags
8
+ attr_reader :query, :type, :title, :description, :tags
14
9
 
15
10
  ANALYZERS = [
16
11
  Mihari::Analyzers::PassiveTotal,
17
- Mihari::Analyzers::SecurityTrails,
12
+ Mihari::Analyzers::SecurityTrails
18
13
  ].freeze
19
14
 
20
15
  def initialize(query, title: nil, description: nil, tags: [])
@@ -37,11 +32,11 @@ module Mihari
37
32
  private
38
33
 
39
34
  def valid_type?
40
- %w(mail).include? type
35
+ %w[mail].include? type
41
36
  end
42
37
 
43
38
  def analyzers
44
- raise InvalidInputError, "#{query}(type: #{type || 'unknown'}) is not supported." unless valid_type?
39
+ raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
45
40
 
46
41
  ANALYZERS.map do |klass|
47
42
  klass.new(query)
@@ -5,12 +5,7 @@ require "securitytrails"
5
5
  module Mihari
6
6
  module Analyzers
7
7
  class SecurityTrails < Base
8
- attr_reader :query
9
- attr_reader :type
10
-
11
- attr_reader :title
12
- attr_reader :description
13
- attr_reader :tags
8
+ attr_reader :query, :type, :title, :description, :tags
14
9
 
15
10
  def initialize(query, title: nil, description: nil, tags: [])
16
11
  super()
@@ -30,7 +25,7 @@ module Mihari
30
25
  private
31
26
 
32
27
  def config_keys
33
- %w(securitytrails_api_key)
28
+ %w[securitytrails_api_key]
34
29
  end
35
30
 
36
31
  def api
@@ -38,7 +33,7 @@ module Mihari
38
33
  end
39
34
 
40
35
  def valid_type?
41
- %w(ip domain mail).include? type
36
+ %w[ip domain mail].include? type
42
37
  end
43
38
 
44
39
  def lookup
@@ -50,28 +45,28 @@ module Mihari
50
45
  when "mail"
51
46
  mail_lookup
52
47
  else
53
- raise InvalidInputError, "#{query}(type: #{type || 'unknown'}) is not supported." unless valid_type?
48
+ raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
54
49
  end
55
50
  end
56
51
 
57
52
  def domain_lookup
58
53
  result = api.history.get_all_dns_history(query, type: "a")
59
- records = result.dig("records") || []
54
+ records = result["records"] || []
60
55
  records.map do |record|
61
- (record.dig("values") || []).map { |value| value.dig("ip") }
56
+ (record["values"] || []).map { |value| value["ip"] }
62
57
  end.flatten.compact.uniq
63
58
  end
64
59
 
65
60
  def ip_lookup
66
- result = api.domains.search( filter: { ipv4: query })
67
- records = result.dig("records") || []
68
- records.map { |record| record.dig("hostname") }.compact.uniq
61
+ result = api.domains.search(filter: {ipv4: query})
62
+ records = result["records"] || []
63
+ records.map { |record| record["hostname"] }.compact.uniq
69
64
  end
70
65
 
71
66
  def mail_lookup
72
- result = api.domains.search( filter: { whois_email: query })
73
- records = result.dig("records") || []
74
- records.map { |record| record.dig("hostname") }.compact.uniq
67
+ result = api.domains.search(filter: {whois_email: query})
68
+ records = result["records"] || []
69
+ records.map { |record| record["hostname"] }.compact.uniq
75
70
  end
76
71
  end
77
72
  end
@@ -5,11 +5,7 @@ require "securitytrails"
5
5
  module Mihari
6
6
  module Analyzers
7
7
  class SecurityTrailsDomainFeed < Base
8
- attr_reader :type
9
-
10
- attr_reader :title
11
- attr_reader :description
12
- attr_reader :tags
8
+ attr_reader :type, :title, :description, :tags
13
9
 
14
10
  def initialize(regexp, type: "registered", title: nil, description: nil, tags: [])
15
11
  super()
@@ -32,7 +28,7 @@ module Mihari
32
28
  private
33
29
 
34
30
  def config_keys
35
- %w(securitytrails_api_key)
31
+ %w[securitytrails_api_key]
36
32
  end
37
33
 
38
34
  def api
@@ -40,7 +36,7 @@ module Mihari
40
36
  end
41
37
 
42
38
  def valid_type?
43
- %w(all new registered).include? type
39
+ %w[all new registered].include? type
44
40
  end
45
41
 
46
42
  def regexp
@@ -5,10 +5,7 @@ require "shodan"
5
5
  module Mihari
6
6
  module Analyzers
7
7
  class Shodan < Base
8
- attr_reader :title
9
- attr_reader :description
10
- attr_reader :query
11
- attr_reader :tags
8
+ attr_reader :title, :description, :query, :tags
12
9
 
13
10
  def initialize(query, title: nil, description: nil, tags: [])
14
11
  super()
@@ -24,9 +21,9 @@ module Mihari
24
21
  return [] unless results || results.empty?
25
22
 
26
23
  results.map do |result|
27
- matches = result.dig("matches") || []
24
+ matches = result["matches"] || []
28
25
  matches.map do |match|
29
- match.dig "ip_str"
26
+ match["ip_str"]
30
27
  end.compact
31
28
  end.flatten.compact.uniq
32
29
  end
@@ -36,7 +33,7 @@ module Mihari
36
33
  PAGE_SIZE = 100
37
34
 
38
35
  def config_keys
39
- %w(shodan_api_key)
36
+ %w[shodan_api_key]
40
37
  end
41
38
 
42
39
  def api
@@ -58,7 +55,11 @@ module Mihari
58
55
  break unless res
59
56
 
60
57
  responses << res
61
- break if res.dig("total").to_i <= page * PAGE_SIZE
58
+ break if res["total"].to_i <= page * PAGE_SIZE
59
+ rescue JSON::ParserError
60
+ # ignore JSON::ParserError
61
+ # ref. https://github.com/ninoseki/mihari/issues/197
62
+ next
62
63
  end
63
64
  responses
64
65
  end
@@ -6,12 +6,7 @@ require "json"
6
6
  module Mihari
7
7
  module Analyzers
8
8
  class Spyse < Base
9
- attr_reader :query
10
- attr_reader :type
11
-
12
- attr_reader :title
13
- attr_reader :description
14
- attr_reader :tags
9
+ attr_reader :query, :type, :title, :description, :tags
15
10
 
16
11
  def initialize(query, title: nil, description: nil, tags: [], type: "domain")
17
12
  super()
@@ -35,7 +30,7 @@ module Mihari
35
30
  end
36
31
 
37
32
  def config_keys
38
- %w(spyse_api_key)
33
+ %w[spyse_api_key]
39
34
  end
40
35
 
41
36
  def api
@@ -43,14 +38,14 @@ module Mihari
43
38
  end
44
39
 
45
40
  def valid_type?
46
- %w(ip domain cert).include? type
41
+ %w[ip domain cert].include? type
47
42
  end
48
43
 
49
44
  def domain_lookup
50
45
  res = api.domain.search(search_params, limit: 100)
51
46
  items = res.dig("data", "items") || []
52
47
  items.map do |item|
53
- item.dig("name")
48
+ item["name"]
54
49
  end.uniq.compact
55
50
  end
56
51
 
@@ -58,7 +53,7 @@ module Mihari
58
53
  res = api.ip.search(search_params, limit: 100)
59
54
  items = res.dig("data", "items") || []
60
55
  items.map do |item|
61
- item.dig("ip")
56
+ item["ip"]
62
57
  end.uniq.compact
63
58
  end
64
59
 
@@ -69,7 +64,7 @@ module Mihari
69
64
  when "ip"
70
65
  ip_lookup
71
66
  else
72
- raise InvalidInputError, "#{query}(type: #{type || 'unknown'}) is not supported." unless valid_type?
67
+ raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
73
68
  end
74
69
  end
75
70
  end