mihari 1.4.1 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/test.yml +44 -0
- data/README.md +6 -7
- data/Rakefile +1 -0
- data/docker/Dockerfile +1 -1
- data/lib/mihari/alert_viewer.rb +3 -3
- data/lib/mihari/analyzers/base.rb +1 -1
- data/lib/mihari/analyzers/basic.rb +3 -4
- data/lib/mihari/analyzers/binaryedge.rb +4 -7
- data/lib/mihari/analyzers/censys.rb +3 -7
- data/lib/mihari/analyzers/circl.rb +3 -5
- data/lib/mihari/analyzers/crtsh.rb +2 -6
- data/lib/mihari/analyzers/dnpedia.rb +3 -6
- data/lib/mihari/analyzers/dnstwister.rb +4 -9
- data/lib/mihari/analyzers/free_text.rb +2 -6
- data/lib/mihari/analyzers/http_hash.rb +3 -11
- data/lib/mihari/analyzers/onyphe.rb +3 -6
- data/lib/mihari/analyzers/otx.rb +4 -9
- data/lib/mihari/analyzers/passive_dns.rb +4 -9
- data/lib/mihari/analyzers/passive_ssl.rb +4 -9
- data/lib/mihari/analyzers/passivetotal.rb +9 -14
- data/lib/mihari/analyzers/pulsedive.rb +7 -12
- data/lib/mihari/analyzers/reverse_whois.rb +4 -9
- data/lib/mihari/analyzers/securitytrails.rb +12 -17
- data/lib/mihari/analyzers/securitytrails_domain_feed.rb +3 -7
- data/lib/mihari/analyzers/shodan.rb +5 -8
- data/lib/mihari/analyzers/spyse.rb +6 -11
- data/lib/mihari/analyzers/ssh_fingerprint.rb +2 -6
- data/lib/mihari/analyzers/urlscan.rb +4 -12
- data/lib/mihari/analyzers/virustotal.rb +6 -11
- data/lib/mihari/analyzers/zoomeye.rb +7 -11
- data/lib/mihari/cli.rb +7 -7
- data/lib/mihari/config.rb +1 -25
- data/lib/mihari/database.rb +1 -1
- data/lib/mihari/emitters/misp.rb +4 -2
- data/lib/mihari/emitters/slack.rb +18 -7
- data/lib/mihari/emitters/the_hive.rb +2 -2
- data/lib/mihari/errors.rb +2 -0
- data/lib/mihari/models/artifact.rb +1 -1
- data/lib/mihari/notifiers/exception_notifier.rb +5 -5
- data/lib/mihari/status.rb +1 -1
- data/lib/mihari/type_checker.rb +4 -4
- data/lib/mihari/version.rb +1 -1
- data/mihari.gemspec +17 -19
- metadata +15 -43
- data/.travis.yml +0 -13
@@ -5,12 +5,7 @@ require "virustotal"
|
|
5
5
|
module Mihari
|
6
6
|
module Analyzers
|
7
7
|
class VirusTotal < Base
|
8
|
-
attr_reader :indicator
|
9
|
-
attr_reader :type
|
10
|
-
|
11
|
-
attr_reader :title
|
12
|
-
attr_reader :description
|
13
|
-
attr_reader :tags
|
8
|
+
attr_reader :indicator, :type, :title, :description, :tags
|
14
9
|
|
15
10
|
def initialize(indicator, title: nil, description: nil, tags: [])
|
16
11
|
super()
|
@@ -30,7 +25,7 @@ module Mihari
|
|
30
25
|
private
|
31
26
|
|
32
27
|
def config_keys
|
33
|
-
%w
|
28
|
+
%w[virustotal_api_key]
|
34
29
|
end
|
35
30
|
|
36
31
|
def api
|
@@ -38,7 +33,7 @@ module Mihari
|
|
38
33
|
end
|
39
34
|
|
40
35
|
def valid_type?
|
41
|
-
%w
|
36
|
+
%w[ip domain].include? type
|
42
37
|
end
|
43
38
|
|
44
39
|
def lookup
|
@@ -48,14 +43,14 @@ module Mihari
|
|
48
43
|
when "ip"
|
49
44
|
ip_lookup
|
50
45
|
else
|
51
|
-
raise InvalidInputError, "#{indicator}(type: #{type ||
|
46
|
+
raise InvalidInputError, "#{indicator}(type: #{type || "unknown"}) is not supported." unless valid_type?
|
52
47
|
end
|
53
48
|
end
|
54
49
|
|
55
50
|
def domain_lookup
|
56
51
|
res = api.domain.resolutions(indicator)
|
57
52
|
|
58
|
-
data = res
|
53
|
+
data = res["data"] || []
|
59
54
|
data.map do |item|
|
60
55
|
item.dig("attributes", "ip_address")
|
61
56
|
end.compact.uniq
|
@@ -64,7 +59,7 @@ module Mihari
|
|
64
59
|
def ip_lookup
|
65
60
|
res = api.ip_address.resolutions(indicator)
|
66
61
|
|
67
|
-
data = res
|
62
|
+
data = res["data"] || []
|
68
63
|
data.map do |item|
|
69
64
|
item.dig("attributes", "host_name")
|
70
65
|
end.compact.uniq
|
@@ -5,11 +5,7 @@ require "zoomeye"
|
|
5
5
|
module Mihari
|
6
6
|
module Analyzers
|
7
7
|
class ZoomEye < Base
|
8
|
-
attr_reader :title
|
9
|
-
attr_reader :description
|
10
|
-
attr_reader :query
|
11
|
-
attr_reader :tags
|
12
|
-
attr_reader :type
|
8
|
+
attr_reader :title, :description, :query, :tags, :type
|
13
9
|
|
14
10
|
def initialize(query, title: nil, description: nil, tags: [], type: "host")
|
15
11
|
super()
|
@@ -37,11 +33,11 @@ module Mihari
|
|
37
33
|
PAGE_SIZE = 10
|
38
34
|
|
39
35
|
def valid_type?
|
40
|
-
%w
|
36
|
+
%w[host web].include? type
|
41
37
|
end
|
42
38
|
|
43
39
|
def config_keys
|
44
|
-
%w
|
40
|
+
%w[zoomeye_password zoomeye_username]
|
45
41
|
end
|
46
42
|
|
47
43
|
def api
|
@@ -50,9 +46,9 @@ module Mihari
|
|
50
46
|
|
51
47
|
def convert_responses(responses)
|
52
48
|
responses.map do |res|
|
53
|
-
matches = res
|
49
|
+
matches = res["matches"] || []
|
54
50
|
matches.map do |match|
|
55
|
-
match
|
51
|
+
match["ip"]
|
56
52
|
end
|
57
53
|
end.flatten.compact.uniq
|
58
54
|
end
|
@@ -69,7 +65,7 @@ module Mihari
|
|
69
65
|
res = _host_lookup(query, page: page)
|
70
66
|
break unless res
|
71
67
|
|
72
|
-
total = res
|
68
|
+
total = res["total"].to_i
|
73
69
|
responses << res
|
74
70
|
break if total <= page * PAGE_SIZE
|
75
71
|
end
|
@@ -88,7 +84,7 @@ module Mihari
|
|
88
84
|
res = _web_lookup(query, page: page)
|
89
85
|
break unless res
|
90
86
|
|
91
|
-
total = res
|
87
|
+
total = res["total"].to_i
|
92
88
|
responses << res
|
93
89
|
break if total <= page * PAGE_SIZE
|
94
90
|
end
|
data/lib/mihari/cli.rb
CHANGED
@@ -259,16 +259,16 @@ module Mihari
|
|
259
259
|
desc "import_from_json", "Give a JSON input via STDIN"
|
260
260
|
def import_from_json(input = nil)
|
261
261
|
with_error_handling do
|
262
|
-
json = input ||
|
262
|
+
json = input || $stdin.gets.chomp
|
263
263
|
raise ArgumentError, "Input not found: please give an input in a JSON format" unless json
|
264
264
|
|
265
265
|
json = parse_as_json(json)
|
266
266
|
raise ArgumentError, "Invalid input format: an input JSON data should have title, description and artifacts key" unless valid_json?(json)
|
267
267
|
|
268
|
-
title = json
|
269
|
-
description = json
|
270
|
-
artifacts = json
|
271
|
-
tags = json
|
268
|
+
title = json["title"]
|
269
|
+
description = json["description"]
|
270
|
+
artifacts = json["artifacts"]
|
271
|
+
tags = json["tags"] || []
|
272
272
|
|
273
273
|
basic = Analyzers::Basic.new(title: title, description: description, artifacts: artifacts, source: "json", tags: tags)
|
274
274
|
basic.run
|
@@ -302,7 +302,7 @@ module Mihari
|
|
302
302
|
no_commands do
|
303
303
|
def with_error_handling
|
304
304
|
yield
|
305
|
-
rescue
|
305
|
+
rescue => e
|
306
306
|
notifier = Notifiers::ExceptionNotifier.new
|
307
307
|
notifier.notify e
|
308
308
|
end
|
@@ -315,7 +315,7 @@ module Mihari
|
|
315
315
|
|
316
316
|
# @return [true, false]
|
317
317
|
def valid_json?(json)
|
318
|
-
%w
|
318
|
+
%w[title description artifacts].all? { |key| json.key? key }
|
319
319
|
end
|
320
320
|
|
321
321
|
def load_configuration
|
data/lib/mihari/config.rb
CHANGED
@@ -4,31 +4,7 @@ require "yaml"
|
|
4
4
|
|
5
5
|
module Mihari
|
6
6
|
class Config
|
7
|
-
attr_accessor :binaryedge_api_key
|
8
|
-
attr_accessor :censys_id
|
9
|
-
attr_accessor :censys_secret
|
10
|
-
attr_accessor :circl_passive_password
|
11
|
-
attr_accessor :circl_passive_username
|
12
|
-
attr_accessor :misp_api_endpoint
|
13
|
-
attr_accessor :misp_api_key
|
14
|
-
attr_accessor :onyphe_api_key
|
15
|
-
attr_accessor :otx_api_key
|
16
|
-
attr_accessor :passivetotal_api_key
|
17
|
-
attr_accessor :passivetotal_username
|
18
|
-
attr_accessor :pulsedive_api_key
|
19
|
-
attr_accessor :securitytrails_api_key
|
20
|
-
attr_accessor :shodan_api_key
|
21
|
-
attr_accessor :slack_channel
|
22
|
-
attr_accessor :slack_webhook_url
|
23
|
-
attr_accessor :spyse_api_key
|
24
|
-
attr_accessor :thehive_api_endpoint
|
25
|
-
attr_accessor :thehive_api_key
|
26
|
-
attr_accessor :urlscan_api_key
|
27
|
-
attr_accessor :virustotal_api_key
|
28
|
-
attr_accessor :zoomeye_password
|
29
|
-
attr_accessor :zoomeye_username
|
30
|
-
|
31
|
-
attr_accessor :database
|
7
|
+
attr_accessor :binaryedge_api_key, :censys_id, :censys_secret, :circl_passive_password, :circl_passive_username, :misp_api_endpoint, :misp_api_key, :onyphe_api_key, :otx_api_key, :passivetotal_api_key, :passivetotal_username, :pulsedive_api_key, :securitytrails_api_key, :shodan_api_key, :slack_channel, :slack_webhook_url, :spyse_api_key, :thehive_api_endpoint, :thehive_api_key, :urlscan_api_key, :virustotal_api_key, :zoomeye_password, :zoomeye_username, :database
|
32
8
|
|
33
9
|
def initialize
|
34
10
|
load_from_env
|
data/lib/mihari/database.rb
CHANGED
data/lib/mihari/emitters/misp.rb
CHANGED
@@ -7,6 +7,8 @@ module Mihari
|
|
7
7
|
module Emitters
|
8
8
|
class MISP < Base
|
9
9
|
def initialize
|
10
|
+
super()
|
11
|
+
|
10
12
|
::MISP.configure do |config|
|
11
13
|
config.api_endpoint = Mihari.config.misp_api_endpoint
|
12
14
|
config.api_key = Mihari.config.misp_api_key
|
@@ -35,7 +37,7 @@ module Mihari
|
|
35
37
|
private
|
36
38
|
|
37
39
|
def config_keys
|
38
|
-
%w
|
40
|
+
%w[misp_api_endpoint misp_api_key]
|
39
41
|
end
|
40
42
|
|
41
43
|
def build_attribute(artifact)
|
@@ -61,7 +63,7 @@ module Mihari
|
|
61
63
|
ip: "ip-dst",
|
62
64
|
mail: "email-dst",
|
63
65
|
url: "url",
|
64
|
-
domain: "domain"
|
66
|
+
domain: "domain"
|
65
67
|
}
|
66
68
|
return table[type] if table.key?(type)
|
67
69
|
|
@@ -19,25 +19,31 @@ module Mihari
|
|
19
19
|
end
|
20
20
|
|
21
21
|
def actions
|
22
|
-
[vt_link, urlscan_link, censys_link].compact
|
22
|
+
[vt_link, urlscan_link, censys_link, shodan_link].compact
|
23
23
|
end
|
24
24
|
|
25
25
|
def vt_link
|
26
26
|
return nil unless _vt_link
|
27
27
|
|
28
|
-
{ type: "button", text: "
|
28
|
+
{ type: "button", text: "VirusTotal", url: _vt_link }
|
29
29
|
end
|
30
30
|
|
31
31
|
def urlscan_link
|
32
32
|
return nil unless _urlscan_link
|
33
33
|
|
34
|
-
{ type: "button", text: "
|
34
|
+
{ type: "button", text: "urlscan.io", url: _urlscan_link }
|
35
35
|
end
|
36
36
|
|
37
37
|
def censys_link
|
38
38
|
return nil unless _censys_link
|
39
39
|
|
40
|
-
{ type: "button", text: "
|
40
|
+
{ type: "button", text: "Censys", url: _censys_link }
|
41
|
+
end
|
42
|
+
|
43
|
+
def shodan_link
|
44
|
+
return nil unless _shodan_link
|
45
|
+
|
46
|
+
{ type: "button", text: "Shodan", url: _shodan_link }
|
41
47
|
end
|
42
48
|
|
43
49
|
# @return [Array]
|
@@ -89,6 +95,11 @@ module Mihari
|
|
89
95
|
end
|
90
96
|
memoize :_censys_link
|
91
97
|
|
98
|
+
def _shodan_link
|
99
|
+
data_type == "ip" ? "https://www.shodan.io/host/#{data}" : nil
|
100
|
+
end
|
101
|
+
memoize :_shodan_link
|
102
|
+
|
92
103
|
# @return [String]
|
93
104
|
def sha256
|
94
105
|
Digest::SHA256.hexdigest data
|
@@ -96,7 +107,7 @@ module Mihari
|
|
96
107
|
|
97
108
|
# @return [String]
|
98
109
|
def defanged_data
|
99
|
-
@defanged_data ||= data.to_s.gsub
|
110
|
+
@defanged_data ||= data.to_s.gsub(/\./, "[.]")
|
100
111
|
end
|
101
112
|
end
|
102
113
|
|
@@ -121,7 +132,7 @@ module Mihari
|
|
121
132
|
[
|
122
133
|
"*#{title}*",
|
123
134
|
"*Desc.*: #{description}",
|
124
|
-
"*Tags*: #{tags.join(', ')}"
|
135
|
+
"*Tags*: #{tags.join(', ')}"
|
125
136
|
].join("\n")
|
126
137
|
end
|
127
138
|
|
@@ -137,7 +148,7 @@ module Mihari
|
|
137
148
|
private
|
138
149
|
|
139
150
|
def config_keys
|
140
|
-
%w
|
151
|
+
%w[slack_webhook_url]
|
141
152
|
end
|
142
153
|
end
|
143
154
|
end
|
@@ -17,7 +17,7 @@ module Mihari
|
|
17
17
|
api.alert.create(
|
18
18
|
title: title,
|
19
19
|
description: description,
|
20
|
-
artifacts: artifacts.map { |artifact| {
|
20
|
+
artifacts: artifacts.map { |artifact| {data: artifact.data, data_type: artifact.data_type, message: description} },
|
21
21
|
tags: tags,
|
22
22
|
type: "external",
|
23
23
|
source: "mihari"
|
@@ -27,7 +27,7 @@ module Mihari
|
|
27
27
|
private
|
28
28
|
|
29
29
|
def config_keys
|
30
|
-
%w
|
30
|
+
%w[thehive_api_endpoint thehive_api_key]
|
31
31
|
end
|
32
32
|
|
33
33
|
def api
|
data/lib/mihari/errors.rb
CHANGED
@@ -19,7 +19,7 @@ module Mihari
|
|
19
19
|
def notify(exception)
|
20
20
|
notify_to_stdout exception
|
21
21
|
|
22
|
-
clean_message = exception.message.tr(
|
22
|
+
clean_message = exception.message.tr("`", "'")
|
23
23
|
attachments = to_attachments(exception, clean_message)
|
24
24
|
notify_to_slack(text: clean_message, attachments: attachments) if @slack.valid?
|
25
25
|
end
|
@@ -51,20 +51,20 @@ module Mihari
|
|
51
51
|
|
52
52
|
def to_fields(clean_message, backtrace)
|
53
53
|
fields = [
|
54
|
-
{
|
55
|
-
{
|
54
|
+
{title: "Exception", value: clean_message},
|
55
|
+
{title: "Hostname", value: hostname}
|
56
56
|
]
|
57
57
|
|
58
58
|
if backtrace
|
59
59
|
formatted_backtrace = format_backtrace(backtrace)
|
60
|
-
fields << {
|
60
|
+
fields << {title: "Backtrace", value: formatted_backtrace}
|
61
61
|
end
|
62
62
|
fields
|
63
63
|
end
|
64
64
|
|
65
65
|
def hostname
|
66
66
|
Socket.gethostname
|
67
|
-
rescue
|
67
|
+
rescue => _e
|
68
68
|
"N/A"
|
69
69
|
end
|
70
70
|
|
data/lib/mihari/status.rb
CHANGED
data/lib/mihari/type_checker.rb
CHANGED
@@ -80,22 +80,22 @@ module Mihari
|
|
80
80
|
|
81
81
|
# @return [true, false]
|
82
82
|
def md5?
|
83
|
-
data.match?
|
83
|
+
data.match?(/^[A-Fa-f0-9]{32}$/)
|
84
84
|
end
|
85
85
|
|
86
86
|
# @return [true, false]
|
87
87
|
def sha1?
|
88
|
-
data.match?
|
88
|
+
data.match?(/^[A-Fa-f0-9]{40}$/)
|
89
89
|
end
|
90
90
|
|
91
91
|
# @return [true, false]
|
92
92
|
def sha256?
|
93
|
-
data.match?
|
93
|
+
data.match?(/^[A-Fa-f0-9]{64}$/)
|
94
94
|
end
|
95
95
|
|
96
96
|
# @return [true, false]
|
97
97
|
def sha512?
|
98
|
-
data.match?
|
98
|
+
data.match?(/^[A-Fa-f0-9]{128}$/)
|
99
99
|
end
|
100
100
|
end
|
101
101
|
end
|
data/lib/mihari/version.rb
CHANGED
data/mihari.gemspec
CHANGED
@@ -1,41 +1,39 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
lib = File.expand_path(
|
3
|
+
lib = File.expand_path("lib", __dir__)
|
4
4
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
5
5
|
require "mihari/version"
|
6
6
|
|
7
7
|
Gem::Specification.new do |spec|
|
8
|
-
spec.name
|
9
|
-
spec.version
|
10
|
-
spec.authors
|
11
|
-
spec.email
|
8
|
+
spec.name = "mihari"
|
9
|
+
spec.version = Mihari::VERSION
|
10
|
+
spec.authors = ["Manabu Niseki"]
|
11
|
+
spec.email = ["manabu.niseki@gmail.com"]
|
12
12
|
|
13
|
-
spec.summary
|
14
|
-
spec.description
|
15
|
-
spec.homepage
|
16
|
-
spec.license
|
13
|
+
spec.summary = "A framework for continuous malicious hosts monitoring."
|
14
|
+
spec.description = "A framework for continuous malicious hosts monitoring."
|
15
|
+
spec.homepage = "https://github.com/ninoseki/mihari"
|
16
|
+
spec.license = "MIT"
|
17
17
|
|
18
18
|
# Specify which files should be added to the gem when it is released.
|
19
19
|
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
20
|
-
spec.files
|
20
|
+
spec.files = Dir.chdir(File.expand_path(__dir__)) do
|
21
21
|
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
22
22
|
end
|
23
|
-
spec.bindir
|
24
|
-
spec.executables
|
23
|
+
spec.bindir = "exe"
|
24
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
25
25
|
spec.require_paths = ["lib"]
|
26
26
|
|
27
|
-
spec.add_development_dependency "bundler", "~> 2.
|
27
|
+
spec.add_development_dependency "bundler", "~> 2.2"
|
28
28
|
spec.add_development_dependency "coveralls", "~> 0.8"
|
29
29
|
spec.add_development_dependency "execjs", "~> 2.7"
|
30
|
-
spec.add_development_dependency "fakefs", "~> 1.
|
31
|
-
spec.add_development_dependency "pre-commit", "~> 0.39"
|
30
|
+
spec.add_development_dependency "fakefs", "~> 1.3"
|
32
31
|
spec.add_development_dependency "rake", "~> 13.0"
|
33
32
|
spec.add_development_dependency "rspec", "~> 3.10"
|
34
|
-
spec.add_development_dependency "
|
35
|
-
spec.add_development_dependency "rubocop-performance", "~> 1.9"
|
33
|
+
spec.add_development_dependency "standard", "~> 1.0"
|
36
34
|
spec.add_development_dependency "timecop", "~> 0.9"
|
37
35
|
spec.add_development_dependency "vcr", "~> 6.0"
|
38
|
-
spec.add_development_dependency "webmock", "~> 3.
|
36
|
+
spec.add_development_dependency "webmock", "~> 3.12"
|
39
37
|
|
40
38
|
spec.add_dependency "active_model_serializers", "~> 0.10"
|
41
39
|
spec.add_dependency "activerecord", "~> 6.1"
|
@@ -64,7 +62,7 @@ Gem::Specification.new do |spec|
|
|
64
62
|
spec.add_dependency "slack-notifier", "~> 2.3"
|
65
63
|
spec.add_dependency "spysex", "~> 0.1"
|
66
64
|
spec.add_dependency "sqlite3", "~> 1.4"
|
67
|
-
spec.add_dependency "thor", "~> 1.
|
65
|
+
spec.add_dependency "thor", "~> 1.1"
|
68
66
|
spec.add_dependency "thread_safe", "~> 0.3"
|
69
67
|
spec.add_dependency "urlscan", "~> 0.6"
|
70
68
|
spec.add_dependency "virustotalx", "~> 1.1"
|