miteru 0.14.7 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/test.yml +68 -0
- data/.gitignore +3 -0
- data/.overcommit.yml +12 -0
- data/.standard.yml +4 -0
- data/README.md +19 -88
- data/docker/Dockerfile +5 -2
- data/lib/miteru/attachement.rb +3 -2
- data/lib/miteru/cli.rb +4 -4
- data/lib/miteru/configuration.rb +19 -0
- data/lib/miteru/crawler.rb +7 -8
- data/lib/miteru/database.rb +73 -0
- data/lib/miteru/downloader.rb +20 -23
- data/lib/miteru/error.rb +1 -0
- data/lib/miteru/feeds/ayashige.rb +2 -2
- data/lib/miteru/feeds/feed.rb +33 -0
- data/lib/miteru/feeds/phishstats.rb +1 -1
- data/lib/miteru/feeds/urlscan_pro.rb +1 -1
- data/lib/miteru/feeds.rb +20 -38
- data/lib/miteru/http_client.rb +7 -7
- data/lib/miteru/kit.rb +41 -16
- data/lib/miteru/mixin.rb +47 -0
- data/lib/miteru/notifier.rb +6 -18
- data/lib/miteru/record.rb +49 -0
- data/lib/miteru/version.rb +1 -1
- data/lib/miteru/website.rb +12 -7
- data/lib/miteru.rb +7 -3
- data/miteru.gemspec +28 -20
- metadata +140 -23
- data/.travis.yml +0 -7
data/lib/miteru/downloader.rb
CHANGED
@@ -6,13 +6,12 @@ require "uri"
|
|
6
6
|
|
7
7
|
module Miteru
|
8
8
|
class Downloader
|
9
|
-
attr_reader :base_dir
|
10
|
-
attr_reader :memo
|
9
|
+
attr_reader :base_dir, :memo
|
11
10
|
|
12
11
|
def initialize(base_dir = "/tmp")
|
13
12
|
@base_dir = base_dir
|
14
13
|
@memo = {}
|
15
|
-
raise ArgumentError, "#{base_dir}
|
14
|
+
raise ArgumentError, "#{base_dir} doesn't exist." unless Dir.exist?(base_dir)
|
16
15
|
end
|
17
16
|
|
18
17
|
def download_kits(kits)
|
@@ -22,23 +21,29 @@ module Miteru
|
|
22
21
|
private
|
23
22
|
|
24
23
|
def download_kit(kit)
|
25
|
-
destination = kit.
|
24
|
+
destination = kit.filepath_to_download
|
25
|
+
|
26
26
|
begin
|
27
|
-
|
28
|
-
hash = sha256(downloaded_filepath)
|
29
|
-
if duplicated?(hash)
|
30
|
-
puts "Do not download #{kit.url} because there is a duplicate file in the directory (SHA256: #{hash})."
|
31
|
-
FileUtils.rm downloaded_filepath
|
32
|
-
else
|
33
|
-
puts "Download #{kit.url} as #{downloaded_filepath}"
|
34
|
-
end
|
27
|
+
downloaded_as = HTTPClient.download(kit.url, destination)
|
35
28
|
rescue Down::Error => e
|
36
29
|
puts "Failed to download: #{kit.url} (#{e})"
|
30
|
+
return
|
37
31
|
end
|
38
|
-
end
|
39
32
|
|
40
|
-
|
41
|
-
|
33
|
+
hash = sha256(downloaded_as)
|
34
|
+
|
35
|
+
ActiveRecord::Base.connection_pool.with_connection do
|
36
|
+
# Remove a downloaded file if it is not unique
|
37
|
+
unless Record.unique_hash?(hash)
|
38
|
+
puts "Don't download #{kit.url}. The same hash is already recorded. (SHA256: #{hash})."
|
39
|
+
FileUtils.rm downloaded_as
|
40
|
+
return
|
41
|
+
end
|
42
|
+
|
43
|
+
# Record a kit in DB
|
44
|
+
Record.create_by_kit_and_hash(kit, hash)
|
45
|
+
puts "Download #{kit.url} as #{downloaded_as}"
|
46
|
+
end
|
42
47
|
end
|
43
48
|
|
44
49
|
def sha256(path)
|
@@ -49,13 +54,5 @@ module Miteru
|
|
49
54
|
memo[path] = hash
|
50
55
|
hash
|
51
56
|
end
|
52
|
-
|
53
|
-
def sha256s
|
54
|
-
Dir.glob("#{base_dir}/*.{zip,rar,7z,tar,gz}").map { |path| sha256(path) }
|
55
|
-
end
|
56
|
-
|
57
|
-
def duplicated?(hash)
|
58
|
-
sha256s.count(hash) > 1
|
59
|
-
end
|
60
57
|
end
|
61
58
|
end
|
data/lib/miteru/error.rb
CHANGED
@@ -10,10 +10,10 @@ module Miteru
|
|
10
10
|
URL = "https://#{HOST}"
|
11
11
|
|
12
12
|
def urls
|
13
|
-
url = url_for("/
|
13
|
+
url = url_for("/api/v1/domains/")
|
14
14
|
res = JSON.parse(get(url))
|
15
15
|
|
16
|
-
domains = res.map { |item| item["
|
16
|
+
domains = res.map { |item| item["fqdn"] }
|
17
17
|
domains.map do |domain|
|
18
18
|
[
|
19
19
|
"https://#{domain}",
|
data/lib/miteru/feeds/feed.rb
CHANGED
@@ -3,10 +3,43 @@
|
|
3
3
|
module Miteru
|
4
4
|
class Feeds
|
5
5
|
class Feed
|
6
|
+
include Mixins::URL
|
7
|
+
|
8
|
+
def source
|
9
|
+
@source ||= self.class.to_s.split("::").last
|
10
|
+
end
|
11
|
+
|
12
|
+
#
|
13
|
+
# Return URLs
|
14
|
+
#
|
15
|
+
# @return [Array<String>] URLs
|
16
|
+
#
|
6
17
|
def urls
|
7
18
|
raise NotImplementedError, "You must implement #{self.class}##{__method__}"
|
8
19
|
end
|
9
20
|
|
21
|
+
#
|
22
|
+
# Return entries
|
23
|
+
#
|
24
|
+
# @return [Array<Miteru::Entry>]
|
25
|
+
#
|
26
|
+
def entries
|
27
|
+
breakdowend_urls.map do |url|
|
28
|
+
Entry.new(url, source)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
#
|
33
|
+
# Return breakdowned URLs
|
34
|
+
#
|
35
|
+
# @return [Array<String>] Breakdowned URLs
|
36
|
+
#
|
37
|
+
def breakdowend_urls
|
38
|
+
urls.select { |url| url.start_with?("http://", "https://") }.map do |url|
|
39
|
+
breakdown(url, Miteru.configuration.directory_traveling?)
|
40
|
+
end.flatten.uniq
|
41
|
+
end
|
42
|
+
|
10
43
|
private
|
11
44
|
|
12
45
|
def get(url)
|
data/lib/miteru/feeds.rb
CHANGED
@@ -8,8 +8,20 @@ require_relative "./feeds/urlscan"
|
|
8
8
|
require_relative "./feeds/urlscan_pro"
|
9
9
|
|
10
10
|
module Miteru
|
11
|
+
class Entry
|
12
|
+
# @return [String]
|
13
|
+
attr_reader :url
|
14
|
+
# @return [String]
|
15
|
+
attr_reader :source
|
16
|
+
|
17
|
+
def initialize(url, source)
|
18
|
+
@url = url
|
19
|
+
@source = source
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
11
23
|
class Feeds
|
12
|
-
IGNORE_EXTENSIONS = %w
|
24
|
+
IGNORE_EXTENSIONS = %w[.htm .html .php .asp .aspx .exe .txt].freeze
|
13
25
|
|
14
26
|
def initialize
|
15
27
|
@feeds = [
|
@@ -21,43 +33,13 @@ module Miteru
|
|
21
33
|
].compact
|
22
34
|
end
|
23
35
|
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
feed.urls.select { |url| url.start_with?("http://", "https://") }
|
32
|
-
end.flatten.uniq
|
33
|
-
|
34
|
-
urls.map { |url| breakdown(url) }.flatten.uniq.sort.each { |url| arr << url }
|
35
|
-
end
|
36
|
-
end
|
37
|
-
|
38
|
-
def breakdown(url)
|
39
|
-
begin
|
40
|
-
uri = URI.parse(url)
|
41
|
-
rescue URI::InvalidURIError => _e
|
42
|
-
return []
|
43
|
-
end
|
44
|
-
|
45
|
-
base = "#{uri.scheme}://#{uri.hostname}"
|
46
|
-
return [base] unless directory_traveling?
|
47
|
-
|
48
|
-
segments = uri.path.split("/")
|
49
|
-
return [base] if segments.length.zero?
|
50
|
-
|
51
|
-
urls = (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join('/')}" }
|
52
|
-
|
53
|
-
urls.reject do |breakdowned_url|
|
54
|
-
# Reject a url which ends with specific extension names
|
55
|
-
invalid_extension? breakdowned_url
|
56
|
-
end
|
57
|
-
end
|
58
|
-
|
59
|
-
def invalid_extension?(url)
|
60
|
-
IGNORE_EXTENSIONS.any? { |ext| url.end_with? ext }
|
36
|
+
#
|
37
|
+
# Returns a list of suspicious entries
|
38
|
+
#
|
39
|
+
# @return [Array<Entry>]
|
40
|
+
#
|
41
|
+
def suspicious_entries
|
42
|
+
@suspicious_entries ||= @feeds.map(&:entries).flatten.uniq(&:url)
|
61
43
|
end
|
62
44
|
end
|
63
45
|
end
|
data/lib/miteru/http_client.rb
CHANGED
@@ -7,7 +7,7 @@ require "uri"
|
|
7
7
|
module Miteru
|
8
8
|
class HTTPClient
|
9
9
|
DEFAULT_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"
|
10
|
-
URLSCAN_UA = "miteru/#{Miteru::VERSION}"
|
10
|
+
URLSCAN_UA = "miteru/#{Miteru::VERSION}".freeze
|
11
11
|
|
12
12
|
attr_reader :ssl_context
|
13
13
|
|
@@ -27,18 +27,18 @@ module Miteru
|
|
27
27
|
options = options.merge default_options
|
28
28
|
|
29
29
|
HTTP.follow
|
30
|
-
|
31
|
-
|
32
|
-
|
30
|
+
.timeout(3)
|
31
|
+
.headers(urlscan_url?(url) ? urlscan_headers : default_headers)
|
32
|
+
.head(url, options)
|
33
33
|
end
|
34
34
|
|
35
35
|
def get(url, options = {})
|
36
36
|
options = options.merge default_options
|
37
37
|
|
38
38
|
HTTP.follow
|
39
|
-
|
40
|
-
|
41
|
-
|
39
|
+
.timeout(write: 2, connect: 5, read: 10)
|
40
|
+
.headers(urlscan_url?(url) ? urlscan_headers : default_headers)
|
41
|
+
.get(url, options)
|
42
42
|
end
|
43
43
|
|
44
44
|
def post(url, options = {})
|
data/lib/miteru/kit.rb
CHANGED
@@ -1,28 +1,43 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require "cgi"
|
4
|
-
require "
|
4
|
+
require "uuidtools"
|
5
|
+
require "uri"
|
5
6
|
|
6
7
|
module Miteru
|
7
8
|
class Kit
|
8
9
|
VALID_EXTENSIONS = Miteru.configuration.valid_extensions
|
9
10
|
VALID_MIME_TYPES = Miteru.configuration.valid_mime_types
|
10
11
|
|
12
|
+
# @return [String]
|
11
13
|
attr_reader :url
|
12
14
|
|
15
|
+
# @return [String]
|
16
|
+
attr_reader :source
|
17
|
+
|
18
|
+
# @return [Integer, nil]
|
13
19
|
attr_reader :status
|
20
|
+
|
21
|
+
# @return [Integer, nil]
|
14
22
|
attr_reader :content_length
|
23
|
+
|
24
|
+
# @return [String, nil]
|
15
25
|
attr_reader :mime_type
|
16
26
|
|
17
|
-
|
27
|
+
# @return [Hash, nil]
|
28
|
+
attr_reader :headers
|
29
|
+
|
30
|
+
def initialize(url, source)
|
18
31
|
@url = url
|
32
|
+
@source = source
|
19
33
|
|
20
34
|
@content_length = nil
|
21
35
|
@mime_type = nil
|
22
36
|
@status = nil
|
37
|
+
@headers = nil
|
23
38
|
end
|
24
39
|
|
25
|
-
def valid
|
40
|
+
def valid?
|
26
41
|
# make a HEAD request for the validation
|
27
42
|
before_validation
|
28
43
|
|
@@ -36,41 +51,50 @@ module Miteru
|
|
36
51
|
end
|
37
52
|
|
38
53
|
def basename
|
39
|
-
File.basename(url)
|
54
|
+
@basename ||= File.basename(url)
|
40
55
|
end
|
41
56
|
|
42
57
|
def filename
|
43
|
-
CGI.unescape
|
58
|
+
@filename ||= CGI.unescape(basename)
|
59
|
+
end
|
60
|
+
|
61
|
+
def filepath_to_download
|
62
|
+
"#{base_dir}/#{filename_to_download}"
|
44
63
|
end
|
45
64
|
|
46
|
-
def
|
47
|
-
|
65
|
+
def downloaded?
|
66
|
+
File.exist?(filepath_to_download)
|
48
67
|
end
|
49
68
|
|
50
69
|
def filesize
|
51
|
-
return nil unless
|
70
|
+
return nil unless downloaded?
|
52
71
|
|
53
|
-
File.size
|
72
|
+
File.size filepath_to_download
|
54
73
|
end
|
55
74
|
|
56
75
|
def filename_with_size
|
57
76
|
return filename unless filesize
|
58
77
|
|
59
|
-
|
78
|
+
kb = (filesize.to_f / 1024.0).ceil
|
79
|
+
"#{filename}(#{kb}KB)"
|
60
80
|
end
|
61
81
|
|
62
|
-
private
|
63
|
-
|
64
82
|
def id
|
65
|
-
@id ||=
|
83
|
+
@id ||= UUIDTools::UUID.random_create.to_s
|
66
84
|
end
|
67
85
|
|
68
86
|
def hostname
|
69
|
-
URI(url).hostname
|
87
|
+
@hostname ||= URI(url).hostname
|
88
|
+
end
|
89
|
+
|
90
|
+
def decoded_url
|
91
|
+
@decoded_url ||= URI.decode_www_form_component(url)
|
70
92
|
end
|
71
93
|
|
72
|
-
|
73
|
-
|
94
|
+
private
|
95
|
+
|
96
|
+
def filename_to_download
|
97
|
+
"#{id}#{extname}"
|
74
98
|
end
|
75
99
|
|
76
100
|
def base_dir
|
@@ -86,6 +110,7 @@ module Miteru
|
|
86
110
|
@content_length = res.content_length
|
87
111
|
@mime_type = res.content_type.mime_type.to_s
|
88
112
|
@status = res.status
|
113
|
+
@headers = res.headers.to_h
|
89
114
|
rescue StandardError
|
90
115
|
# do nothing
|
91
116
|
end
|
data/lib/miteru/mixin.rb
ADDED
@@ -0,0 +1,47 @@
|
|
1
|
+
module Miteru
|
2
|
+
module Mixins
|
3
|
+
module URL
|
4
|
+
IGNORE_EXTENSIONS = %w[.htm .html .php .asp .aspx .exe .txt].freeze
|
5
|
+
|
6
|
+
#
|
7
|
+
# Validate extension of a URL
|
8
|
+
#
|
9
|
+
# @param [String] url
|
10
|
+
#
|
11
|
+
# @return [Boolean]
|
12
|
+
#
|
13
|
+
def invalid_extension?(url)
|
14
|
+
IGNORE_EXTENSIONS.any? { |ext| url.end_with? ext }
|
15
|
+
end
|
16
|
+
|
17
|
+
#
|
18
|
+
# Breakdown a URL into URLs
|
19
|
+
#
|
20
|
+
# @param [String] url
|
21
|
+
# @param [Boolean] enable_directory_traveling
|
22
|
+
#
|
23
|
+
# @return [Array<String>]
|
24
|
+
#
|
25
|
+
def breakdown(url, enable_directory_traveling)
|
26
|
+
begin
|
27
|
+
uri = URI.parse(url)
|
28
|
+
rescue URI::InvalidURIError => _e
|
29
|
+
return []
|
30
|
+
end
|
31
|
+
|
32
|
+
base = "#{uri.scheme}://#{uri.hostname}"
|
33
|
+
return [base] unless enable_directory_traveling
|
34
|
+
|
35
|
+
segments = uri.path.split("/")
|
36
|
+
return [base] if segments.length.zero?
|
37
|
+
|
38
|
+
urls = (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join("/")}" }
|
39
|
+
|
40
|
+
urls.reject do |breakdowned_url|
|
41
|
+
# Reject a url which ends with specific extension names
|
42
|
+
invalid_extension? breakdowned_url
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
data/lib/miteru/notifier.rb
CHANGED
@@ -7,31 +7,19 @@ module Miteru
|
|
7
7
|
class Notifier
|
8
8
|
def notify(url:, kits:, message:)
|
9
9
|
attachement = Attachement.new(url)
|
10
|
-
kits = kits.select(&:
|
10
|
+
kits = kits.select(&:downloaded?)
|
11
11
|
|
12
|
-
if
|
13
|
-
notifier = Slack::Notifier.new(slack_webhook_url, channel: slack_channel)
|
14
|
-
notifier.post(text: message, attachments: attachement.to_a)
|
12
|
+
if notifiable? && kits.any?
|
13
|
+
notifier = Slack::Notifier.new(Miteru.configuration.slack_webhook_url, channel: Miteru.configuration.slack_channel)
|
14
|
+
notifier.post(text: message.capitalize, attachments: attachement.to_a)
|
15
15
|
end
|
16
16
|
|
17
17
|
message = message.colorize(:light_red) if kits.any?
|
18
18
|
puts "#{url}: #{message}"
|
19
19
|
end
|
20
20
|
|
21
|
-
def
|
22
|
-
slack_webhook_url? && Miteru.configuration.post_to_slack?
|
23
|
-
end
|
24
|
-
|
25
|
-
def slack_webhook_url
|
26
|
-
ENV.fetch "SLACK_WEBHOOK_URL"
|
27
|
-
end
|
28
|
-
|
29
|
-
def slack_channel
|
30
|
-
ENV.fetch "SLACK_CHANNEL", "#general"
|
31
|
-
end
|
32
|
-
|
33
|
-
def slack_webhook_url?
|
34
|
-
ENV.key? "SLACK_WEBHOOK_URL"
|
21
|
+
def notifiable?
|
22
|
+
Miteru.configuration.slack_webhook_url? && Miteru.configuration.post_to_slack?
|
35
23
|
end
|
36
24
|
end
|
37
25
|
end
|