miteru 0.13.0 → 0.14.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +1 -0
- data/README.md +2 -0
- data/lib/miteru/configuration.rb +4 -0
- data/lib/miteru/feeds.rb +5 -0
- data/lib/miteru/feeds/phishing_database.rb +20 -0
- data/lib/miteru/feeds/phishstats.rb +28 -0
- data/lib/miteru/http_client.rb +14 -1
- data/lib/miteru/kit.rb +29 -15
- data/lib/miteru/version.rb +1 -1
- data/lib/miteru/website.rb +31 -5
- data/miteru.gemspec +6 -6
- metadata +20 -18
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 89102143e9e405545e94b50da7b81b90c85ae9612c8c914d4b50b6d09ca2e023
|
4
|
+
data.tar.gz: 2d6e60e6301425b9ceb8ed7cf103caab144f2ddab4fdd3872ef3f332a6667e37
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3df145d910400efc069d07c718943560a064b5d9fff849fb695795bf49b1d30681e6c31946172f2cd0e7508ca46ded7ff93d2935e48fc6dc970e1817d6e6348c
|
7
|
+
data.tar.gz: 70057c7c2451f4631bfda279431e40dfc86bc4931caa555141e607d0a2d73e5308bd0458ddc9046b1e0ccbe0329829ab8568202e8671c68146e5b6b4b6688705
|
data/.travis.yml
CHANGED
data/README.md
CHANGED
@@ -17,6 +17,8 @@ Miteru is an experimental phishing kit detection tool.
|
|
17
17
|
- [URLhaus feed via urlscan.io](https://urlscan.io/search/#URLHaus)
|
18
18
|
- urlscan.io phish feed (available for Pro users)
|
19
19
|
- [Ayashige feed](https://github.com/ninoseki/ayashige)
|
20
|
+
- [Phishing Database feed](https://github.com/mitchellkrogza/Phishing.Database)
|
21
|
+
- [PhishStats feed](https://phishstats.info/)
|
20
22
|
- It checks each phishy URL whether it enables directory listing and contains a phishing kit (compressed file) or not.
|
21
23
|
- Note: compressed file = `*.zip`, `*.rar`, `*.7z`, `*.tar` and `*.gz`.
|
22
24
|
|
data/lib/miteru/configuration.rb
CHANGED
@@ -28,6 +28,8 @@ module Miteru
|
|
28
28
|
# @return [Boolean]
|
29
29
|
attr_accessor :verbose
|
30
30
|
|
31
|
+
attr_reader :valid_extensions
|
32
|
+
|
31
33
|
def initialize
|
32
34
|
@auto_download = false
|
33
35
|
@ayashige = false
|
@@ -37,6 +39,8 @@ module Miteru
|
|
37
39
|
@size = 100
|
38
40
|
@threads = Parallel.processor_count
|
39
41
|
@verbose = false
|
42
|
+
|
43
|
+
@valid_extensions = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
|
40
44
|
end
|
41
45
|
|
42
46
|
def auto_download?
|
data/lib/miteru/feeds.rb
CHANGED
@@ -1,6 +1,8 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require_relative "./feeds/feed"
|
4
|
+
require_relative "./feeds/phishing_database"
|
5
|
+
require_relative "./feeds/phishstats"
|
4
6
|
require_relative "./feeds/ayashige"
|
5
7
|
require_relative "./feeds/urlscan"
|
6
8
|
require_relative "./feeds/urlscan_pro"
|
@@ -11,6 +13,8 @@ module Miteru
|
|
11
13
|
|
12
14
|
def initialize
|
13
15
|
@feeds = [
|
16
|
+
PhishingDatabase.new,
|
17
|
+
PhishStats.new,
|
14
18
|
UrlScan.new(Miteru.configuration.size),
|
15
19
|
UrlScanPro.new,
|
16
20
|
Miteru.configuration.ayashige? ? Ayashige.new : nil
|
@@ -45,6 +49,7 @@ module Miteru
|
|
45
49
|
return [base] if segments.length.zero?
|
46
50
|
|
47
51
|
urls = (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join('/')}" }
|
52
|
+
|
48
53
|
urls.reject do |breakdowned_url|
|
49
54
|
# Reject a url which ends with specific extension names
|
50
55
|
invalid_extension? breakdowned_url
|
@@ -0,0 +1,20 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "json"
|
4
|
+
require "uri"
|
5
|
+
|
6
|
+
module Miteru
|
7
|
+
class Feeds
|
8
|
+
class PhishingDatabase < Feed
|
9
|
+
URL = "https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/phishing-links-NEW-today.txt"
|
10
|
+
|
11
|
+
def urls
|
12
|
+
body = get(URL)
|
13
|
+
body.to_s.lines.map(&:chomp)
|
14
|
+
rescue HTTPResponseError, HTTP::Error, JSON::ParserError => e
|
15
|
+
puts "Failed to load phishing database feed (#{e})"
|
16
|
+
[]
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "json"
|
4
|
+
require "uri"
|
5
|
+
|
6
|
+
module Miteru
|
7
|
+
class Feeds
|
8
|
+
class PhishStats < Feed
|
9
|
+
URL = "https://phishstats.info:2096/api/phishing?_sort=-id&size=100"
|
10
|
+
|
11
|
+
def urls
|
12
|
+
json = JSON.parse(get(URL))
|
13
|
+
json.map do |entry|
|
14
|
+
entry.dig("url")
|
15
|
+
end
|
16
|
+
rescue HTTPResponseError, HTTP::Error, JSON::ParserError => e
|
17
|
+
puts "Failed to load PhishStats feed (#{e})"
|
18
|
+
[]
|
19
|
+
end
|
20
|
+
|
21
|
+
private
|
22
|
+
|
23
|
+
def url_for(path)
|
24
|
+
URI(URL + path)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
data/lib/miteru/http_client.rb
CHANGED
@@ -18,11 +18,20 @@ module Miteru
|
|
18
18
|
end
|
19
19
|
|
20
20
|
def download(url, destination)
|
21
|
-
down = Down::Http.new(default_options) { |client| client.headers(default_headers) }
|
21
|
+
down = Down::Http.new(**default_options) { |client| client.headers(**default_headers) }
|
22
22
|
down.download(url, destination: destination)
|
23
23
|
destination
|
24
24
|
end
|
25
25
|
|
26
|
+
def head(url, options = {})
|
27
|
+
options = options.merge default_options
|
28
|
+
|
29
|
+
HTTP.follow
|
30
|
+
.timeout(3)
|
31
|
+
.headers(urlscan_url?(url) ? urlscan_headers : default_headers)
|
32
|
+
.head(url, options)
|
33
|
+
end
|
34
|
+
|
26
35
|
def get(url, options = {})
|
27
36
|
options = options.merge default_options
|
28
37
|
|
@@ -48,6 +57,10 @@ module Miteru
|
|
48
57
|
def post(url, options = {})
|
49
58
|
new.post url, options
|
50
59
|
end
|
60
|
+
|
61
|
+
def head(url, options = {})
|
62
|
+
new.head url, options
|
63
|
+
end
|
51
64
|
end
|
52
65
|
|
53
66
|
private
|
data/lib/miteru/kit.rb
CHANGED
@@ -5,37 +5,32 @@ require "securerandom"
|
|
5
5
|
|
6
6
|
module Miteru
|
7
7
|
class Kit
|
8
|
-
VALID_EXTENSIONS =
|
8
|
+
VALID_EXTENSIONS = Miteru.configuration.valid_extensions
|
9
9
|
|
10
|
-
attr_reader :
|
10
|
+
attr_reader :url
|
11
11
|
|
12
|
-
def initialize(
|
13
|
-
@
|
14
|
-
@link = link.start_with?("/") ? link[1..-1] : link
|
12
|
+
def initialize(url)
|
13
|
+
@url = url
|
15
14
|
end
|
16
15
|
|
17
|
-
def valid
|
18
|
-
|
16
|
+
def valid?;
|
17
|
+
valid_ext? && reachable_and_archive_file?
|
19
18
|
end
|
20
19
|
|
21
20
|
def extname
|
22
|
-
return ".tar.gz" if
|
21
|
+
return ".tar.gz" if url.end_with?("tar.gz")
|
23
22
|
|
24
|
-
File.extname(
|
23
|
+
File.extname(url)
|
25
24
|
end
|
26
25
|
|
27
26
|
def basename
|
28
|
-
File.basename(
|
27
|
+
File.basename(url)
|
29
28
|
end
|
30
29
|
|
31
30
|
def filename
|
32
31
|
CGI.unescape basename
|
33
32
|
end
|
34
33
|
|
35
|
-
def url
|
36
|
-
"#{base_url}/#{basename}"
|
37
|
-
end
|
38
|
-
|
39
34
|
def download_filepath
|
40
35
|
"#{base_dir}/#{download_filename}"
|
41
36
|
end
|
@@ -59,7 +54,7 @@ module Miteru
|
|
59
54
|
end
|
60
55
|
|
61
56
|
def hostname
|
62
|
-
URI(
|
57
|
+
URI(url).hostname
|
63
58
|
end
|
64
59
|
|
65
60
|
def download_filename
|
@@ -69,5 +64,24 @@ module Miteru
|
|
69
64
|
def base_dir
|
70
65
|
@base_dir ||= Miteru.configuration.download_to
|
71
66
|
end
|
67
|
+
|
68
|
+
def valid_ext?
|
69
|
+
VALID_EXTENSIONS.include? extname
|
70
|
+
end
|
71
|
+
|
72
|
+
def reachable?(response)
|
73
|
+
response.status.success?
|
74
|
+
end
|
75
|
+
|
76
|
+
def archive_file?(response)
|
77
|
+
!response.content_type.mime_type.to_s.start_with? "text/"
|
78
|
+
end
|
79
|
+
|
80
|
+
def reachable_and_archive_file?
|
81
|
+
res = HTTPClient.head(url)
|
82
|
+
reachable?(res) && archive_file?(res)
|
83
|
+
rescue StandardError
|
84
|
+
false
|
85
|
+
end
|
72
86
|
end
|
73
87
|
end
|
data/lib/miteru/version.rb
CHANGED
data/lib/miteru/website.rb
CHANGED
@@ -4,7 +4,10 @@ require "oga"
|
|
4
4
|
|
5
5
|
module Miteru
|
6
6
|
class Website
|
7
|
+
VALID_EXTENSIONS = Miteru.configuration.valid_extensions
|
8
|
+
|
7
9
|
attr_reader :url
|
10
|
+
|
8
11
|
def initialize(url)
|
9
12
|
@url = url
|
10
13
|
end
|
@@ -15,7 +18,7 @@ module Miteru
|
|
15
18
|
|
16
19
|
def kits
|
17
20
|
@kits ||= links.map do |link|
|
18
|
-
kit = Kit.new(
|
21
|
+
kit = Kit.new(link)
|
19
22
|
kit.valid? ? kit : nil
|
20
23
|
end.compact
|
21
24
|
end
|
@@ -33,7 +36,7 @@ module Miteru
|
|
33
36
|
end
|
34
37
|
|
35
38
|
def has_kits?
|
36
|
-
|
39
|
+
kits?
|
37
40
|
rescue Addressable::URI::InvalidURIError, ArgumentError, Encoding::CompatibilityError, HTTP::Error, LL::ParserError, OpenSSL::SSL::SSLError => _e
|
38
41
|
false
|
39
42
|
end
|
@@ -46,6 +49,10 @@ module Miteru
|
|
46
49
|
"It might contain #{noun}: #{filename_with_sizes}."
|
47
50
|
end
|
48
51
|
|
52
|
+
def links
|
53
|
+
(href_links + possible_file_links).compact.uniq
|
54
|
+
end
|
55
|
+
|
49
56
|
private
|
50
57
|
|
51
58
|
def response
|
@@ -66,12 +73,31 @@ module Miteru
|
|
66
73
|
nil
|
67
74
|
end
|
68
75
|
|
69
|
-
def
|
70
|
-
if doc
|
71
|
-
doc.css("a").map { |a| a.get("href") }.compact
|
76
|
+
def href_links
|
77
|
+
if doc && ok? && index?
|
78
|
+
doc.css("a").map { |a| a.get("href") }.compact.map do |href|
|
79
|
+
href = href.start_with?("/") ? href : "/#{href}"
|
80
|
+
url + href
|
81
|
+
end
|
72
82
|
else
|
73
83
|
[]
|
74
84
|
end
|
85
|
+
rescue Addressable::URI::InvalidURIError, ArgumentError, Encoding::CompatibilityError, HTTP::Error, LL::ParserError, OpenSSL::SSL::SSLError => _e
|
86
|
+
[]
|
87
|
+
end
|
88
|
+
|
89
|
+
def possible_file_links
|
90
|
+
uri = URI.parse(url)
|
91
|
+
|
92
|
+
segments = uri.path.split("/")
|
93
|
+
return [] if segments.length.zero?
|
94
|
+
|
95
|
+
last = segments.last
|
96
|
+
VALID_EXTENSIONS.map do |ext|
|
97
|
+
new_segments = segments[0..-2] + ["#{last}#{ext}"]
|
98
|
+
uri.path = new_segments.join("/")
|
99
|
+
uri.to_s
|
100
|
+
end
|
75
101
|
end
|
76
102
|
end
|
77
103
|
end
|
data/miteru.gemspec
CHANGED
@@ -24,18 +24,18 @@ Gem::Specification.new do |spec|
|
|
24
24
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
25
25
|
spec.require_paths = ["lib"]
|
26
26
|
|
27
|
-
spec.add_development_dependency "bundler", "~> 2.
|
27
|
+
spec.add_development_dependency "bundler", "~> 2.1"
|
28
28
|
spec.add_development_dependency "coveralls", "~> 0.8"
|
29
29
|
spec.add_development_dependency "glint", "~> 0.1"
|
30
30
|
spec.add_development_dependency "rake", "~> 13.0"
|
31
31
|
spec.add_development_dependency "rspec", "~> 3.9"
|
32
|
-
spec.add_development_dependency "vcr", "~>
|
33
|
-
spec.add_development_dependency "webmock", "~> 3.
|
32
|
+
spec.add_development_dependency "vcr", "~> 6.0"
|
33
|
+
spec.add_development_dependency "webmock", "~> 3.8"
|
34
34
|
|
35
35
|
spec.add_dependency "colorize", "~> 0.8"
|
36
|
-
spec.add_dependency "down", "~> 5.
|
37
|
-
spec.add_dependency "http", "~> 4.
|
38
|
-
spec.add_dependency "oga", "~> 3.
|
36
|
+
spec.add_dependency "down", "~> 5.1"
|
37
|
+
spec.add_dependency "http", "~> 4.4"
|
38
|
+
spec.add_dependency "oga", "~> 3.2"
|
39
39
|
spec.add_dependency "parallel", "~> 1.19"
|
40
40
|
spec.add_dependency "slack-notifier", "~> 2.3"
|
41
41
|
spec.add_dependency "thor", "~> 1.0"
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: miteru
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.14.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Manabu Niseki
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-06-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -16,14 +16,14 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '2.
|
19
|
+
version: '2.1'
|
20
20
|
type: :development
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '2.
|
26
|
+
version: '2.1'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: coveralls
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -86,28 +86,28 @@ dependencies:
|
|
86
86
|
requirements:
|
87
87
|
- - "~>"
|
88
88
|
- !ruby/object:Gem::Version
|
89
|
-
version: '
|
89
|
+
version: '6.0'
|
90
90
|
type: :development
|
91
91
|
prerelease: false
|
92
92
|
version_requirements: !ruby/object:Gem::Requirement
|
93
93
|
requirements:
|
94
94
|
- - "~>"
|
95
95
|
- !ruby/object:Gem::Version
|
96
|
-
version: '
|
96
|
+
version: '6.0'
|
97
97
|
- !ruby/object:Gem::Dependency
|
98
98
|
name: webmock
|
99
99
|
requirement: !ruby/object:Gem::Requirement
|
100
100
|
requirements:
|
101
101
|
- - "~>"
|
102
102
|
- !ruby/object:Gem::Version
|
103
|
-
version: '3.
|
103
|
+
version: '3.8'
|
104
104
|
type: :development
|
105
105
|
prerelease: false
|
106
106
|
version_requirements: !ruby/object:Gem::Requirement
|
107
107
|
requirements:
|
108
108
|
- - "~>"
|
109
109
|
- !ruby/object:Gem::Version
|
110
|
-
version: '3.
|
110
|
+
version: '3.8'
|
111
111
|
- !ruby/object:Gem::Dependency
|
112
112
|
name: colorize
|
113
113
|
requirement: !ruby/object:Gem::Requirement
|
@@ -128,42 +128,42 @@ dependencies:
|
|
128
128
|
requirements:
|
129
129
|
- - "~>"
|
130
130
|
- !ruby/object:Gem::Version
|
131
|
-
version: '5.
|
131
|
+
version: '5.1'
|
132
132
|
type: :runtime
|
133
133
|
prerelease: false
|
134
134
|
version_requirements: !ruby/object:Gem::Requirement
|
135
135
|
requirements:
|
136
136
|
- - "~>"
|
137
137
|
- !ruby/object:Gem::Version
|
138
|
-
version: '5.
|
138
|
+
version: '5.1'
|
139
139
|
- !ruby/object:Gem::Dependency
|
140
140
|
name: http
|
141
141
|
requirement: !ruby/object:Gem::Requirement
|
142
142
|
requirements:
|
143
143
|
- - "~>"
|
144
144
|
- !ruby/object:Gem::Version
|
145
|
-
version: '4.
|
145
|
+
version: '4.4'
|
146
146
|
type: :runtime
|
147
147
|
prerelease: false
|
148
148
|
version_requirements: !ruby/object:Gem::Requirement
|
149
149
|
requirements:
|
150
150
|
- - "~>"
|
151
151
|
- !ruby/object:Gem::Version
|
152
|
-
version: '4.
|
152
|
+
version: '4.4'
|
153
153
|
- !ruby/object:Gem::Dependency
|
154
154
|
name: oga
|
155
155
|
requirement: !ruby/object:Gem::Requirement
|
156
156
|
requirements:
|
157
157
|
- - "~>"
|
158
158
|
- !ruby/object:Gem::Version
|
159
|
-
version: '3.
|
159
|
+
version: '3.2'
|
160
160
|
type: :runtime
|
161
161
|
prerelease: false
|
162
162
|
version_requirements: !ruby/object:Gem::Requirement
|
163
163
|
requirements:
|
164
164
|
- - "~>"
|
165
165
|
- !ruby/object:Gem::Version
|
166
|
-
version: '3.
|
166
|
+
version: '3.2'
|
167
167
|
- !ruby/object:Gem::Dependency
|
168
168
|
name: parallel
|
169
169
|
requirement: !ruby/object:Gem::Requirement
|
@@ -249,6 +249,8 @@ files:
|
|
249
249
|
- lib/miteru/feeds.rb
|
250
250
|
- lib/miteru/feeds/ayashige.rb
|
251
251
|
- lib/miteru/feeds/feed.rb
|
252
|
+
- lib/miteru/feeds/phishing_database.rb
|
253
|
+
- lib/miteru/feeds/phishstats.rb
|
252
254
|
- lib/miteru/feeds/urlscan.rb
|
253
255
|
- lib/miteru/feeds/urlscan_pro.rb
|
254
256
|
- lib/miteru/http_client.rb
|
@@ -262,7 +264,7 @@ homepage: https://github.com/ninoseki/miteru
|
|
262
264
|
licenses:
|
263
265
|
- MIT
|
264
266
|
metadata: {}
|
265
|
-
post_install_message:
|
267
|
+
post_install_message:
|
266
268
|
rdoc_options: []
|
267
269
|
require_paths:
|
268
270
|
- lib
|
@@ -277,8 +279,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
277
279
|
- !ruby/object:Gem::Version
|
278
280
|
version: '0'
|
279
281
|
requirements: []
|
280
|
-
rubygems_version: 3.
|
281
|
-
signing_key:
|
282
|
+
rubygems_version: 3.1.2
|
283
|
+
signing_key:
|
282
284
|
specification_version: 4
|
283
285
|
summary: An experimental phishing kit detector
|
284
286
|
test_files: []
|