miteru 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +2 -1
- data/lib/miteru/cli.rb +6 -6
- data/lib/miteru/crawler.rb +12 -2
- data/lib/miteru/version.rb +1 -1
- data/lib/miteru/website.rb +7 -7
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 48d47d401250750c0ffa68ccf5f58602a1e0fa4918b0b812aebcd0c60cdfe484
|
4
|
+
data.tar.gz: 9acbfd9d4465c25ddb20186fe57a2d0a84ba74ba24fa2e55b2b2103b5a4d12f9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7690a2497203522b7c9469656a5e1ea9fd2e72671555548a1d0833841d8b3bce34a60c22b2e9597b94add35342dbae0b2cb3fd58fdb3ec08b9e16375f1ca6044
|
7
|
+
data.tar.gz: 27d5aef80aa2bae48091bfaf398fae956e7226384414bac4505af3752090a722c1be1b421d3260773dd40d7d7c2dc65281b5555bbade267f83554d267371cd85
|
data/README.md
CHANGED
@@ -13,7 +13,8 @@ Miteru is an experimental phishing kit detection tool.
|
|
13
13
|
- [urlscan.io](https://urlscan.io/search/#certstream-suspicious)
|
14
14
|
- [OpenPhish community feed](https://openphish.com/feed.txt)
|
15
15
|
- [PhishTank feed](http://data.phishtank.com/data/online-valid.csv)
|
16
|
-
- It checks a suspicious URL whether it has a directory listing and contains a phishing kit (
|
16
|
+
- It checks a suspicious URL whether it has a directory listing and contains a phishing kit (compressed file) or not.
|
17
|
+
- Note: compressed file = `*.zip`, `*.rar`, `*.7z`, `*.tar` and `*.gz`.
|
17
18
|
|
18
19
|
## Installation
|
19
20
|
|
data/lib/miteru/cli.rb
CHANGED
@@ -8,7 +8,7 @@ require "thor"
|
|
8
8
|
|
9
9
|
module Miteru
|
10
10
|
class CLI < Thor
|
11
|
-
method_option :auto_download, type: :boolean, default: false, desc: "Enable or disable auto-download of
|
11
|
+
method_option :auto_download, type: :boolean, default: false, desc: "Enable or disable auto-download of compressed file(s)"
|
12
12
|
method_option :directory_traveling, type: :boolean, default: false, desc: "Enable or disable directory traveling"
|
13
13
|
method_option :download_to, type: :string, default: "/tmp", desc: "Directory to download file(s)"
|
14
14
|
method_option :post_to_slack, type: :boolean, default: false, desc: "Post a message to Slack if it detects a phishing kit"
|
@@ -26,16 +26,16 @@ module Miteru
|
|
26
26
|
websites.each do |website|
|
27
27
|
next unless website.has_kit?
|
28
28
|
|
29
|
-
message = "#{website.url}: it might contain phishing kit(s) (#{website.
|
29
|
+
message = "#{website.url}: it might contain phishing kit(s) (#{website.compressed_files.join(', ')})."
|
30
30
|
puts message.colorize(:light_red)
|
31
|
-
post_to_slack(message) if options[:post_to_slack] && valid_slack_setting?
|
32
|
-
|
31
|
+
post_to_slack(website.message) if options[:post_to_slack] && valid_slack_setting?
|
32
|
+
download_compressed_files(website.url, website.compressed_files, options[:download_to]) if options[:auto_download]
|
33
33
|
end
|
34
34
|
end
|
35
35
|
|
36
36
|
no_commands do
|
37
|
-
def
|
38
|
-
|
37
|
+
def download_compressed_files(url, compressed_files, base_dir)
|
38
|
+
compressed_files.each do |path|
|
39
39
|
target_url = "#{url}/#{path}"
|
40
40
|
begin
|
41
41
|
download_file_path = HTTPClient.download(target_url, base_dir)
|
data/lib/miteru/crawler.rb
CHANGED
@@ -29,17 +29,23 @@ module Miteru
|
|
29
29
|
url = "#{URLSCAN_ENDPOINT}/search/?q=certstream-suspicious&size=#{size}"
|
30
30
|
res = JSON.parse(get(url))
|
31
31
|
res["results"].map { |result| result.dig("task", "url") }
|
32
|
+
rescue HTTPResponseError => _
|
33
|
+
[]
|
32
34
|
end
|
33
35
|
|
34
36
|
def openphish_feed
|
35
37
|
res = get("#{OPENPHISH_ENDPOINT}/feed.txt")
|
36
38
|
res.lines.map(&:chomp)
|
39
|
+
rescue HTTPResponseError => _
|
40
|
+
[]
|
37
41
|
end
|
38
42
|
|
39
43
|
def phishtank_feed
|
40
44
|
res = get("#{PHISHTANK_ENDPOINT}/data/online-valid.csv")
|
41
45
|
table = CSV.parse(res, headers: true)
|
42
46
|
table.map { |row| row["url"] }
|
47
|
+
rescue HTTPResponseError => _
|
48
|
+
[]
|
43
49
|
end
|
44
50
|
|
45
51
|
def breakdown(url)
|
@@ -62,11 +68,15 @@ module Miteru
|
|
62
68
|
end
|
63
69
|
|
64
70
|
def suspicious_urls
|
65
|
-
|
66
|
-
|
71
|
+
@suspicious_urls ||= [].tap do |arr|
|
72
|
+
urls = (urlscan_feed + openphish_feed + phishtank_feed)
|
73
|
+
urls.map { |url| breakdown(url) }.flatten.uniq.sort.each { |url| arr << url }
|
74
|
+
end
|
67
75
|
end
|
68
76
|
|
69
77
|
def execute
|
78
|
+
puts "Loaded #{suspicious_urls.length} URLs to crawl." if verbose
|
79
|
+
|
70
80
|
pool = Thread.pool(threads)
|
71
81
|
websites = []
|
72
82
|
|
data/lib/miteru/version.rb
CHANGED
data/lib/miteru/website.rb
CHANGED
@@ -14,10 +14,10 @@ module Miteru
|
|
14
14
|
doc.at_css("title")&.text
|
15
15
|
end
|
16
16
|
|
17
|
-
def
|
18
|
-
@
|
17
|
+
def compressed_files
|
18
|
+
@compressed_files ||= doc.css("a").map do |a|
|
19
19
|
href = a.get("href")
|
20
|
-
|
20
|
+
[".zip", ".rar", ".7z", ".tar", ".gz"].any? { |ext| href&.end_with? ext } ? href : nil
|
21
21
|
end.compact.map do |href|
|
22
22
|
href.start_with?("/") ? href[1..-1] : href
|
23
23
|
end
|
@@ -31,12 +31,12 @@ module Miteru
|
|
31
31
|
title == "Index of /"
|
32
32
|
end
|
33
33
|
|
34
|
-
def
|
35
|
-
!
|
34
|
+
def compressed_files?
|
35
|
+
!compressed_files.empty?
|
36
36
|
end
|
37
37
|
|
38
38
|
def has_kit?
|
39
|
-
@has_kit ||= ok? && index? &&
|
39
|
+
@has_kit ||= ok? && index? && compressed_files?
|
40
40
|
end
|
41
41
|
|
42
42
|
def build
|
@@ -46,7 +46,7 @@ module Miteru
|
|
46
46
|
def unbuild
|
47
47
|
@doc = nil
|
48
48
|
@response = nil
|
49
|
-
@
|
49
|
+
@compressed_files = nil
|
50
50
|
end
|
51
51
|
|
52
52
|
private
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: miteru
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.9.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Manabu Niseki
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-10-
|
11
|
+
date: 2018-10-16 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|