mihari 1.3.1 → 1.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/test.yml +44 -0
- data/README.md +23 -12
- data/Rakefile +1 -0
- data/docker/Dockerfile +3 -2
- data/{screenshots → images}/alert.png +0 -0
- data/{screenshots → images}/eyecatch.png +0 -0
- data/images/logo.png +0 -0
- data/{screenshots → images}/misp.png +0 -0
- data/{screenshots → images}/slack.png +0 -0
- data/lib/mihari/alert_viewer.rb +3 -3
- data/lib/mihari/analyzers/base.rb +1 -1
- data/lib/mihari/analyzers/basic.rb +3 -4
- data/lib/mihari/analyzers/binaryedge.rb +4 -7
- data/lib/mihari/analyzers/censys.rb +3 -7
- data/lib/mihari/analyzers/circl.rb +3 -5
- data/lib/mihari/analyzers/crtsh.rb +2 -6
- data/lib/mihari/analyzers/dnpedia.rb +3 -6
- data/lib/mihari/analyzers/dnstwister.rb +4 -9
- data/lib/mihari/analyzers/free_text.rb +2 -6
- data/lib/mihari/analyzers/http_hash.rb +3 -11
- data/lib/mihari/analyzers/onyphe.rb +3 -6
- data/lib/mihari/analyzers/otx.rb +4 -9
- data/lib/mihari/analyzers/passive_dns.rb +4 -9
- data/lib/mihari/analyzers/passive_ssl.rb +4 -9
- data/lib/mihari/analyzers/passivetotal.rb +9 -14
- data/lib/mihari/analyzers/pulsedive.rb +7 -12
- data/lib/mihari/analyzers/reverse_whois.rb +4 -9
- data/lib/mihari/analyzers/securitytrails.rb +12 -17
- data/lib/mihari/analyzers/securitytrails_domain_feed.rb +3 -7
- data/lib/mihari/analyzers/shodan.rb +9 -8
- data/lib/mihari/analyzers/spyse.rb +6 -11
- data/lib/mihari/analyzers/ssh_fingerprint.rb +2 -6
- data/lib/mihari/analyzers/urlscan.rb +21 -9
- data/lib/mihari/analyzers/virustotal.rb +6 -11
- data/lib/mihari/analyzers/zoomeye.rb +7 -11
- data/lib/mihari/cli.rb +14 -7
- data/lib/mihari/config.rb +1 -25
- data/lib/mihari/database.rb +1 -1
- data/lib/mihari/emitters/misp.rb +4 -2
- data/lib/mihari/emitters/slack.rb +18 -7
- data/lib/mihari/emitters/the_hive.rb +2 -2
- data/lib/mihari/errors.rb +2 -0
- data/lib/mihari/models/artifact.rb +1 -1
- data/lib/mihari/notifiers/exception_notifier.rb +5 -5
- data/lib/mihari/status.rb +1 -1
- data/lib/mihari/type_checker.rb +4 -4
- data/lib/mihari/version.rb +1 -1
- data/mihari.gemspec +23 -24
- metadata +44 -57
- data/.travis.yml +0 -13
@@ -5,10 +5,7 @@ require "onyphe"
|
|
5
5
|
module Mihari
|
6
6
|
module Analyzers
|
7
7
|
class Onyphe < Base
|
8
|
-
attr_reader :title
|
9
|
-
attr_reader :description
|
10
|
-
attr_reader :query
|
11
|
-
attr_reader :tags
|
8
|
+
attr_reader :title, :description, :query, :tags
|
12
9
|
|
13
10
|
def initialize(query, title: nil, description: nil, tags: [])
|
14
11
|
super()
|
@@ -35,7 +32,7 @@ module Mihari
|
|
35
32
|
PAGE_SIZE = 10
|
36
33
|
|
37
34
|
def config_keys
|
38
|
-
%w
|
35
|
+
%w[onyphe_api_key]
|
39
36
|
end
|
40
37
|
|
41
38
|
def api
|
@@ -51,7 +48,7 @@ module Mihari
|
|
51
48
|
(1..Float::INFINITY).each do |page|
|
52
49
|
res = search_with_page(query, page: page)
|
53
50
|
responses << res
|
54
|
-
total = res
|
51
|
+
total = res["total"].to_i
|
55
52
|
break if total <= page * PAGE_SIZE
|
56
53
|
end
|
57
54
|
responses
|
data/lib/mihari/analyzers/otx.rb
CHANGED
@@ -5,12 +5,7 @@ require "otx_ruby"
|
|
5
5
|
module Mihari
|
6
6
|
module Analyzers
|
7
7
|
class OTX < Base
|
8
|
-
attr_reader :query
|
9
|
-
attr_reader :type
|
10
|
-
|
11
|
-
attr_reader :title
|
12
|
-
attr_reader :description
|
13
|
-
attr_reader :tags
|
8
|
+
attr_reader :query, :type, :title, :description, :tags
|
14
9
|
|
15
10
|
def initialize(query, title: nil, description: nil, tags: [])
|
16
11
|
super()
|
@@ -30,7 +25,7 @@ module Mihari
|
|
30
25
|
private
|
31
26
|
|
32
27
|
def config_keys
|
33
|
-
%w
|
28
|
+
%w[otx_api_key]
|
34
29
|
end
|
35
30
|
|
36
31
|
def domain_client
|
@@ -42,7 +37,7 @@ module Mihari
|
|
42
37
|
end
|
43
38
|
|
44
39
|
def valid_type?
|
45
|
-
%w
|
40
|
+
%w[ip domain].include? type
|
46
41
|
end
|
47
42
|
|
48
43
|
def lookup
|
@@ -52,7 +47,7 @@ module Mihari
|
|
52
47
|
when "ip"
|
53
48
|
ip_lookup
|
54
49
|
else
|
55
|
-
raise InvalidInputError, "#{query}(type: #{type ||
|
50
|
+
raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
|
56
51
|
end
|
57
52
|
end
|
58
53
|
|
@@ -5,12 +5,7 @@ require "parallel"
|
|
5
5
|
module Mihari
|
6
6
|
module Analyzers
|
7
7
|
class PassiveDNS < Base
|
8
|
-
attr_reader :query
|
9
|
-
attr_reader :type
|
10
|
-
|
11
|
-
attr_reader :title
|
12
|
-
attr_reader :description
|
13
|
-
attr_reader :tags
|
8
|
+
attr_reader :query, :type, :title, :description, :tags
|
14
9
|
|
15
10
|
ANALYZERS = [
|
16
11
|
Mihari::Analyzers::CIRCL,
|
@@ -18,7 +13,7 @@ module Mihari
|
|
18
13
|
Mihari::Analyzers::PassiveTotal,
|
19
14
|
Mihari::Analyzers::Pulsedive,
|
20
15
|
Mihari::Analyzers::SecurityTrails,
|
21
|
-
Mihari::Analyzers::VirusTotal
|
16
|
+
Mihari::Analyzers::VirusTotal
|
22
17
|
].freeze
|
23
18
|
|
24
19
|
def initialize(query, title: nil, description: nil, tags: [])
|
@@ -41,11 +36,11 @@ module Mihari
|
|
41
36
|
private
|
42
37
|
|
43
38
|
def valid_type?
|
44
|
-
%w
|
39
|
+
%w[ip domain].include? type
|
45
40
|
end
|
46
41
|
|
47
42
|
def analyzers
|
48
|
-
raise InvalidInputError, "#{query}(type: #{type ||
|
43
|
+
raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
|
49
44
|
|
50
45
|
ANALYZERS.map do |klass|
|
51
46
|
klass.new(query)
|
@@ -5,16 +5,11 @@ require "parallel"
|
|
5
5
|
module Mihari
|
6
6
|
module Analyzers
|
7
7
|
class PassiveSSL < Base
|
8
|
-
attr_reader :query
|
9
|
-
attr_reader :type
|
10
|
-
|
11
|
-
attr_reader :title
|
12
|
-
attr_reader :description
|
13
|
-
attr_reader :tags
|
8
|
+
attr_reader :query, :type, :title, :description, :tags
|
14
9
|
|
15
10
|
ANALYZERS = [
|
16
11
|
Mihari::Analyzers::CIRCL,
|
17
|
-
Mihari::Analyzers::PassiveTotal
|
12
|
+
Mihari::Analyzers::PassiveTotal
|
18
13
|
].freeze
|
19
14
|
|
20
15
|
def initialize(query, title: nil, description: nil, tags: [])
|
@@ -37,11 +32,11 @@ module Mihari
|
|
37
32
|
private
|
38
33
|
|
39
34
|
def valid_type?
|
40
|
-
%w
|
35
|
+
%w[sha1].include? type
|
41
36
|
end
|
42
37
|
|
43
38
|
def analyzers
|
44
|
-
raise InvalidInputError, "#{query}(type: #{type ||
|
39
|
+
raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
|
45
40
|
|
46
41
|
ANALYZERS.map do |klass|
|
47
42
|
klass.new(query)
|
@@ -5,12 +5,7 @@ require "passivetotal"
|
|
5
5
|
module Mihari
|
6
6
|
module Analyzers
|
7
7
|
class PassiveTotal < Base
|
8
|
-
attr_reader :query
|
9
|
-
attr_reader :type
|
10
|
-
|
11
|
-
attr_reader :title
|
12
|
-
attr_reader :description
|
13
|
-
attr_reader :tags
|
8
|
+
attr_reader :query, :type, :title, :description, :tags
|
14
9
|
|
15
10
|
def initialize(query, title: nil, description: nil, tags: [])
|
16
11
|
super()
|
@@ -30,7 +25,7 @@ module Mihari
|
|
30
25
|
private
|
31
26
|
|
32
27
|
def config_keys
|
33
|
-
%w
|
28
|
+
%w[passivetotal_username passivetotal_api_key]
|
34
29
|
end
|
35
30
|
|
36
31
|
def api
|
@@ -38,7 +33,7 @@ module Mihari
|
|
38
33
|
end
|
39
34
|
|
40
35
|
def valid_type?
|
41
|
-
%w
|
36
|
+
%w[ip domain mail].include? type
|
42
37
|
end
|
43
38
|
|
44
39
|
def lookup
|
@@ -52,28 +47,28 @@ module Mihari
|
|
52
47
|
when "hash"
|
53
48
|
ssl_lookup
|
54
49
|
else
|
55
|
-
raise InvalidInputError, "#{query}(type: #{type ||
|
50
|
+
raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
|
56
51
|
end
|
57
52
|
end
|
58
53
|
|
59
54
|
def passive_dns_lookup
|
60
55
|
res = api.dns.passive_unique(query)
|
61
|
-
res
|
56
|
+
res["results"] || []
|
62
57
|
end
|
63
58
|
|
64
59
|
def reverse_whois_lookup
|
65
60
|
res = api.whois.search(query: query, field: "email")
|
66
|
-
results = res
|
61
|
+
results = res["results"] || []
|
67
62
|
results.map do |result|
|
68
|
-
result
|
63
|
+
result["domain"]
|
69
64
|
end.flatten.compact.uniq
|
70
65
|
end
|
71
66
|
|
72
67
|
def ssl_lookup
|
73
68
|
res = api.ssl.history(query)
|
74
|
-
results = res
|
69
|
+
results = res["results"] || []
|
75
70
|
results.map do |result|
|
76
|
-
result
|
71
|
+
result["ipAddresses"]
|
77
72
|
end.flatten.compact.uniq
|
78
73
|
end
|
79
74
|
end
|
@@ -5,12 +5,7 @@ require "pulsedive"
|
|
5
5
|
module Mihari
|
6
6
|
module Analyzers
|
7
7
|
class Pulsedive < Base
|
8
|
-
attr_reader :query
|
9
|
-
attr_reader :type
|
10
|
-
|
11
|
-
attr_reader :title
|
12
|
-
attr_reader :description
|
13
|
-
attr_reader :tags
|
8
|
+
attr_reader :query, :type, :title, :description, :tags
|
14
9
|
|
15
10
|
def initialize(query, title: nil, description: nil, tags: [])
|
16
11
|
super()
|
@@ -30,7 +25,7 @@ module Mihari
|
|
30
25
|
private
|
31
26
|
|
32
27
|
def config_keys
|
33
|
-
%w
|
28
|
+
%w[pulsedive_api_key]
|
34
29
|
end
|
35
30
|
|
36
31
|
def api
|
@@ -38,18 +33,18 @@ module Mihari
|
|
38
33
|
end
|
39
34
|
|
40
35
|
def valid_type?
|
41
|
-
%w
|
36
|
+
%w[ip domain].include? type
|
42
37
|
end
|
43
38
|
|
44
39
|
def lookup
|
45
|
-
raise InvalidInputError, "#{query}(type: #{type ||
|
40
|
+
raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
|
46
41
|
|
47
42
|
indicator = api.indicator.get_by_value(query)
|
48
|
-
iid = indicator
|
43
|
+
iid = indicator["iid"]
|
49
44
|
|
50
45
|
properties = api.indicator.get_properties_by_id(iid)
|
51
|
-
(properties
|
52
|
-
property
|
46
|
+
(properties["dns"] || []).map do |property|
|
47
|
+
property["value"] if ["A", "PTR"].include?(property["name"])
|
53
48
|
end.compact
|
54
49
|
end
|
55
50
|
end
|
@@ -5,16 +5,11 @@ require "parallel"
|
|
5
5
|
module Mihari
|
6
6
|
module Analyzers
|
7
7
|
class ReveseWhois < Base
|
8
|
-
attr_reader :query
|
9
|
-
attr_reader :type
|
10
|
-
|
11
|
-
attr_reader :title
|
12
|
-
attr_reader :description
|
13
|
-
attr_reader :tags
|
8
|
+
attr_reader :query, :type, :title, :description, :tags
|
14
9
|
|
15
10
|
ANALYZERS = [
|
16
11
|
Mihari::Analyzers::PassiveTotal,
|
17
|
-
Mihari::Analyzers::SecurityTrails
|
12
|
+
Mihari::Analyzers::SecurityTrails
|
18
13
|
].freeze
|
19
14
|
|
20
15
|
def initialize(query, title: nil, description: nil, tags: [])
|
@@ -37,11 +32,11 @@ module Mihari
|
|
37
32
|
private
|
38
33
|
|
39
34
|
def valid_type?
|
40
|
-
%w
|
35
|
+
%w[mail].include? type
|
41
36
|
end
|
42
37
|
|
43
38
|
def analyzers
|
44
|
-
raise InvalidInputError, "#{query}(type: #{type ||
|
39
|
+
raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
|
45
40
|
|
46
41
|
ANALYZERS.map do |klass|
|
47
42
|
klass.new(query)
|
@@ -5,12 +5,7 @@ require "securitytrails"
|
|
5
5
|
module Mihari
|
6
6
|
module Analyzers
|
7
7
|
class SecurityTrails < Base
|
8
|
-
attr_reader :query
|
9
|
-
attr_reader :type
|
10
|
-
|
11
|
-
attr_reader :title
|
12
|
-
attr_reader :description
|
13
|
-
attr_reader :tags
|
8
|
+
attr_reader :query, :type, :title, :description, :tags
|
14
9
|
|
15
10
|
def initialize(query, title: nil, description: nil, tags: [])
|
16
11
|
super()
|
@@ -30,7 +25,7 @@ module Mihari
|
|
30
25
|
private
|
31
26
|
|
32
27
|
def config_keys
|
33
|
-
%w
|
28
|
+
%w[securitytrails_api_key]
|
34
29
|
end
|
35
30
|
|
36
31
|
def api
|
@@ -38,7 +33,7 @@ module Mihari
|
|
38
33
|
end
|
39
34
|
|
40
35
|
def valid_type?
|
41
|
-
%w
|
36
|
+
%w[ip domain mail].include? type
|
42
37
|
end
|
43
38
|
|
44
39
|
def lookup
|
@@ -50,28 +45,28 @@ module Mihari
|
|
50
45
|
when "mail"
|
51
46
|
mail_lookup
|
52
47
|
else
|
53
|
-
raise InvalidInputError, "#{query}(type: #{type ||
|
48
|
+
raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
|
54
49
|
end
|
55
50
|
end
|
56
51
|
|
57
52
|
def domain_lookup
|
58
53
|
result = api.history.get_all_dns_history(query, type: "a")
|
59
|
-
records = result
|
54
|
+
records = result["records"] || []
|
60
55
|
records.map do |record|
|
61
|
-
(record
|
56
|
+
(record["values"] || []).map { |value| value["ip"] }
|
62
57
|
end.flatten.compact.uniq
|
63
58
|
end
|
64
59
|
|
65
60
|
def ip_lookup
|
66
|
-
result = api.domains.search(
|
67
|
-
records = result
|
68
|
-
records.map { |record| record
|
61
|
+
result = api.domains.search(filter: {ipv4: query})
|
62
|
+
records = result["records"] || []
|
63
|
+
records.map { |record| record["hostname"] }.compact.uniq
|
69
64
|
end
|
70
65
|
|
71
66
|
def mail_lookup
|
72
|
-
result = api.domains.search(
|
73
|
-
records = result
|
74
|
-
records.map { |record| record
|
67
|
+
result = api.domains.search(filter: {whois_email: query})
|
68
|
+
records = result["records"] || []
|
69
|
+
records.map { |record| record["hostname"] }.compact.uniq
|
75
70
|
end
|
76
71
|
end
|
77
72
|
end
|
@@ -5,11 +5,7 @@ require "securitytrails"
|
|
5
5
|
module Mihari
|
6
6
|
module Analyzers
|
7
7
|
class SecurityTrailsDomainFeed < Base
|
8
|
-
attr_reader :type
|
9
|
-
|
10
|
-
attr_reader :title
|
11
|
-
attr_reader :description
|
12
|
-
attr_reader :tags
|
8
|
+
attr_reader :type, :title, :description, :tags
|
13
9
|
|
14
10
|
def initialize(regexp, type: "registered", title: nil, description: nil, tags: [])
|
15
11
|
super()
|
@@ -32,7 +28,7 @@ module Mihari
|
|
32
28
|
private
|
33
29
|
|
34
30
|
def config_keys
|
35
|
-
%w
|
31
|
+
%w[securitytrails_api_key]
|
36
32
|
end
|
37
33
|
|
38
34
|
def api
|
@@ -40,7 +36,7 @@ module Mihari
|
|
40
36
|
end
|
41
37
|
|
42
38
|
def valid_type?
|
43
|
-
%w
|
39
|
+
%w[all new registered].include? type
|
44
40
|
end
|
45
41
|
|
46
42
|
def regexp
|
@@ -5,10 +5,7 @@ require "shodan"
|
|
5
5
|
module Mihari
|
6
6
|
module Analyzers
|
7
7
|
class Shodan < Base
|
8
|
-
attr_reader :title
|
9
|
-
attr_reader :description
|
10
|
-
attr_reader :query
|
11
|
-
attr_reader :tags
|
8
|
+
attr_reader :title, :description, :query, :tags
|
12
9
|
|
13
10
|
def initialize(query, title: nil, description: nil, tags: [])
|
14
11
|
super()
|
@@ -24,9 +21,9 @@ module Mihari
|
|
24
21
|
return [] unless results || results.empty?
|
25
22
|
|
26
23
|
results.map do |result|
|
27
|
-
matches = result
|
24
|
+
matches = result["matches"] || []
|
28
25
|
matches.map do |match|
|
29
|
-
match
|
26
|
+
match["ip_str"]
|
30
27
|
end.compact
|
31
28
|
end.flatten.compact.uniq
|
32
29
|
end
|
@@ -36,7 +33,7 @@ module Mihari
|
|
36
33
|
PAGE_SIZE = 100
|
37
34
|
|
38
35
|
def config_keys
|
39
|
-
%w
|
36
|
+
%w[shodan_api_key]
|
40
37
|
end
|
41
38
|
|
42
39
|
def api
|
@@ -58,7 +55,11 @@ module Mihari
|
|
58
55
|
break unless res
|
59
56
|
|
60
57
|
responses << res
|
61
|
-
break if res
|
58
|
+
break if res["total"].to_i <= page * PAGE_SIZE
|
59
|
+
rescue JSON::ParserError
|
60
|
+
# ignore JSON::ParserError
|
61
|
+
# ref. https://github.com/ninoseki/mihari/issues/197
|
62
|
+
next
|
62
63
|
end
|
63
64
|
responses
|
64
65
|
end
|
@@ -6,12 +6,7 @@ require "json"
|
|
6
6
|
module Mihari
|
7
7
|
module Analyzers
|
8
8
|
class Spyse < Base
|
9
|
-
attr_reader :query
|
10
|
-
attr_reader :type
|
11
|
-
|
12
|
-
attr_reader :title
|
13
|
-
attr_reader :description
|
14
|
-
attr_reader :tags
|
9
|
+
attr_reader :query, :type, :title, :description, :tags
|
15
10
|
|
16
11
|
def initialize(query, title: nil, description: nil, tags: [], type: "domain")
|
17
12
|
super()
|
@@ -35,7 +30,7 @@ module Mihari
|
|
35
30
|
end
|
36
31
|
|
37
32
|
def config_keys
|
38
|
-
%w
|
33
|
+
%w[spyse_api_key]
|
39
34
|
end
|
40
35
|
|
41
36
|
def api
|
@@ -43,14 +38,14 @@ module Mihari
|
|
43
38
|
end
|
44
39
|
|
45
40
|
def valid_type?
|
46
|
-
%w
|
41
|
+
%w[ip domain cert].include? type
|
47
42
|
end
|
48
43
|
|
49
44
|
def domain_lookup
|
50
45
|
res = api.domain.search(search_params, limit: 100)
|
51
46
|
items = res.dig("data", "items") || []
|
52
47
|
items.map do |item|
|
53
|
-
item
|
48
|
+
item["name"]
|
54
49
|
end.uniq.compact
|
55
50
|
end
|
56
51
|
|
@@ -58,7 +53,7 @@ module Mihari
|
|
58
53
|
res = api.ip.search(search_params, limit: 100)
|
59
54
|
items = res.dig("data", "items") || []
|
60
55
|
items.map do |item|
|
61
|
-
item
|
56
|
+
item["ip"]
|
62
57
|
end.uniq.compact
|
63
58
|
end
|
64
59
|
|
@@ -69,7 +64,7 @@ module Mihari
|
|
69
64
|
when "ip"
|
70
65
|
ip_lookup
|
71
66
|
else
|
72
|
-
raise InvalidInputError, "#{query}(type: #{type ||
|
67
|
+
raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
|
73
68
|
end
|
74
69
|
end
|
75
70
|
end
|