miteru 0.14.2 → 0.14.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/docker/Dockerfile +1 -1
- data/exe/miteru +2 -1
- data/lib/miteru/cli.rb +8 -0
- data/lib/miteru/configuration.rb +7 -0
- data/lib/miteru/feeds.rb +1 -13
- data/lib/miteru/feeds/phishing_database.rb +1 -1
- data/lib/miteru/http_client.rb +1 -1
- data/lib/miteru/kit.rb +47 -15
- data/lib/miteru/version.rb +1 -1
- data/lib/miteru/website.rb +31 -29
- data/miteru.gemspec +4 -4
- data/renovate.json +5 -0
- metadata +14 -13
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c1ae4b442c2963ff387cc1bf3bf6390a393c2f8ba416276f2dc0cc48f2ceb86b
|
4
|
+
data.tar.gz: ce3baa2e515837cf722a4bd90650f558a337cde2e2dd2ecef976620f7d20eddb
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 40afc8ff440ffad5be4e4b7efdb7670c6df1a9e05d503930424e53a6c57fcfd3270239f39a92de2ccac149e8bc6175c87833183ad822d900d526490f067c06dc
|
7
|
+
data.tar.gz: 2ad37a6b2ebfaf78451ba35fa70d826ad95202cf077fdba5ed02cbebd8c74104c6f7f2e49da1ea26a9d42890bbd0b4d7b76e06787cab3697ba43f441a5825230
|
data/docker/Dockerfile
CHANGED
data/exe/miteru
CHANGED
data/lib/miteru/cli.rb
CHANGED
data/lib/miteru/configuration.rb
CHANGED
@@ -28,6 +28,10 @@ module Miteru
|
|
28
28
|
# @return [Boolean]
|
29
29
|
attr_accessor :verbose
|
30
30
|
|
31
|
+
attr_reader :valid_extensions
|
32
|
+
|
33
|
+
attr_reader :valid_mime_types
|
34
|
+
|
31
35
|
def initialize
|
32
36
|
@auto_download = false
|
33
37
|
@ayashige = false
|
@@ -37,6 +41,9 @@ module Miteru
|
|
37
41
|
@size = 100
|
38
42
|
@threads = Parallel.processor_count
|
39
43
|
@verbose = false
|
44
|
+
|
45
|
+
@valid_extensions = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
|
46
|
+
@valid_mime_types = ["application/zip", "application/vnd.rar", "application/x-7z-compressed", "application/x-tar", "application/gzip"]
|
40
47
|
end
|
41
48
|
|
42
49
|
def auto_download?
|
data/lib/miteru/feeds.rb
CHANGED
@@ -10,7 +10,6 @@ require_relative "./feeds/urlscan_pro"
|
|
10
10
|
module Miteru
|
11
11
|
class Feeds
|
12
12
|
IGNORE_EXTENSIONS = %w(.htm .html .php .asp .aspx .exe .txt).freeze
|
13
|
-
VALID_EXTENSIONS = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
|
14
13
|
|
15
14
|
def initialize
|
16
15
|
@feeds = [
|
@@ -49,18 +48,7 @@ module Miteru
|
|
49
48
|
segments = uri.path.split("/")
|
50
49
|
return [base] if segments.length.zero?
|
51
50
|
|
52
|
-
urls = (0...segments.length).map
|
53
|
-
breakdowned_url = "#{base}#{segments[0..idx].join('/')}"
|
54
|
-
breakdown = [breakdowned_url]
|
55
|
-
if idx > 0 && idx < segments.length
|
56
|
-
next if segments[idx].nil? || invalid_extension?(segments[idx])
|
57
|
-
|
58
|
-
VALID_EXTENSIONS.each do |ext|
|
59
|
-
breakdown << "#{base}#{segments[0..idx - 1].join('/')}/#{segments[idx]}#{ext}"
|
60
|
-
end
|
61
|
-
end
|
62
|
-
breakdown
|
63
|
-
end.flatten.compact
|
51
|
+
urls = (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join('/')}" }
|
64
52
|
|
65
53
|
urls.reject do |breakdowned_url|
|
66
54
|
# Reject a url which ends with specific extension names
|
@@ -6,7 +6,7 @@ require "uri"
|
|
6
6
|
module Miteru
|
7
7
|
class Feeds
|
8
8
|
class PhishingDatabase < Feed
|
9
|
-
URL = "https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/phishing-links-
|
9
|
+
URL = "https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/phishing-links-ACTIVE-NOW.txt"
|
10
10
|
|
11
11
|
def urls
|
12
12
|
body = get(URL)
|
data/lib/miteru/http_client.rb
CHANGED
@@ -18,7 +18,7 @@ module Miteru
|
|
18
18
|
end
|
19
19
|
|
20
20
|
def download(url, destination)
|
21
|
-
down = Down::Http.new(default_options) { |client| client.headers(default_headers) }
|
21
|
+
down = Down::Http.new(**default_options) { |client| client.headers(**default_headers) }
|
22
22
|
down.download(url, destination: destination)
|
23
23
|
destination
|
24
24
|
end
|
data/lib/miteru/kit.rb
CHANGED
@@ -5,37 +5,44 @@ require "securerandom"
|
|
5
5
|
|
6
6
|
module Miteru
|
7
7
|
class Kit
|
8
|
-
VALID_EXTENSIONS =
|
8
|
+
VALID_EXTENSIONS = Miteru.configuration.valid_extensions
|
9
|
+
VALID_MIME_TYPES = Miteru.configuration.valid_mime_types
|
9
10
|
|
10
|
-
attr_reader :
|
11
|
+
attr_reader :url
|
11
12
|
|
12
|
-
|
13
|
-
|
14
|
-
|
13
|
+
attr_reader :status
|
14
|
+
attr_reader :content_length
|
15
|
+
attr_reader :mime_type
|
16
|
+
|
17
|
+
def initialize(url)
|
18
|
+
@url = url
|
19
|
+
|
20
|
+
@content_length = nil
|
21
|
+
@mime_type = nil
|
22
|
+
@status = nil
|
15
23
|
end
|
16
24
|
|
17
|
-
def valid
|
18
|
-
|
25
|
+
def valid?;
|
26
|
+
# make a HEAD request for the validation
|
27
|
+
before_validation
|
28
|
+
|
29
|
+
valid_ext? && reachable? && valid_mime_type? && valid_content_length?
|
19
30
|
end
|
20
31
|
|
21
32
|
def extname
|
22
|
-
return ".tar.gz" if
|
33
|
+
return ".tar.gz" if url.end_with?("tar.gz")
|
23
34
|
|
24
|
-
File.extname(
|
35
|
+
File.extname(url)
|
25
36
|
end
|
26
37
|
|
27
38
|
def basename
|
28
|
-
File.basename(
|
39
|
+
File.basename(url)
|
29
40
|
end
|
30
41
|
|
31
42
|
def filename
|
32
43
|
CGI.unescape basename
|
33
44
|
end
|
34
45
|
|
35
|
-
def url
|
36
|
-
"#{base_url}/#{basename}"
|
37
|
-
end
|
38
|
-
|
39
46
|
def download_filepath
|
40
47
|
"#{base_dir}/#{download_filename}"
|
41
48
|
end
|
@@ -59,7 +66,7 @@ module Miteru
|
|
59
66
|
end
|
60
67
|
|
61
68
|
def hostname
|
62
|
-
URI(
|
69
|
+
URI(url).hostname
|
63
70
|
end
|
64
71
|
|
65
72
|
def download_filename
|
@@ -69,5 +76,30 @@ module Miteru
|
|
69
76
|
def base_dir
|
70
77
|
@base_dir ||= Miteru.configuration.download_to
|
71
78
|
end
|
79
|
+
|
80
|
+
def valid_ext?
|
81
|
+
VALID_EXTENSIONS.include? extname
|
82
|
+
end
|
83
|
+
|
84
|
+
def before_validation
|
85
|
+
res = HTTPClient.head(url)
|
86
|
+
@content_length = res.content_length
|
87
|
+
@mime_type = res.content_type.mime_type.to_s
|
88
|
+
@status = res.status
|
89
|
+
rescue StandardError
|
90
|
+
# do nothing
|
91
|
+
end
|
92
|
+
|
93
|
+
def reachable?
|
94
|
+
status&.success?
|
95
|
+
end
|
96
|
+
|
97
|
+
def valid_mime_type?
|
98
|
+
VALID_MIME_TYPES.include? mime_type
|
99
|
+
end
|
100
|
+
|
101
|
+
def valid_content_length?
|
102
|
+
content_length.to_i > 0
|
103
|
+
end
|
72
104
|
end
|
73
105
|
end
|
data/lib/miteru/version.rb
CHANGED
data/lib/miteru/website.rb
CHANGED
@@ -4,9 +4,10 @@ require "oga"
|
|
4
4
|
|
5
5
|
module Miteru
|
6
6
|
class Website
|
7
|
-
VALID_EXTENSIONS =
|
7
|
+
VALID_EXTENSIONS = Miteru.configuration.valid_extensions
|
8
8
|
|
9
9
|
attr_reader :url
|
10
|
+
|
10
11
|
def initialize(url)
|
11
12
|
@url = url
|
12
13
|
end
|
@@ -16,25 +17,12 @@ module Miteru
|
|
16
17
|
end
|
17
18
|
|
18
19
|
def kits
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
link = url.split("/").last
|
23
|
-
base_url = url.split("/")[0..-2].join("/")
|
24
|
-
kit = Kit.new(base_url: base_url, link: link)
|
25
|
-
return kit.valid? ? [kit] : []
|
26
|
-
end
|
27
|
-
|
28
|
-
links.map do |link|
|
29
|
-
kit = Kit.new(base_url: url, link: link.to_s)
|
20
|
+
@kits ||= links.map do |link|
|
21
|
+
kit = Kit.new(link)
|
30
22
|
kit.valid? ? kit : nil
|
31
23
|
end.compact
|
32
24
|
end
|
33
25
|
|
34
|
-
def ext?
|
35
|
-
VALID_EXTENSIONS.any? { |ext| url.end_with?(ext) }
|
36
|
-
end
|
37
|
-
|
38
26
|
def ok?
|
39
27
|
response.code == 200
|
40
28
|
end
|
@@ -48,9 +36,7 @@ module Miteru
|
|
48
36
|
end
|
49
37
|
|
50
38
|
def has_kits?
|
51
|
-
|
52
|
-
|
53
|
-
ok? && index? && kits?
|
39
|
+
kits?
|
54
40
|
rescue Addressable::URI::InvalidURIError, ArgumentError, Encoding::CompatibilityError, HTTP::Error, LL::ParserError, OpenSSL::SSL::SSLError => _e
|
55
41
|
false
|
56
42
|
end
|
@@ -63,19 +49,16 @@ module Miteru
|
|
63
49
|
"It might contain #{noun}: #{filename_with_sizes}."
|
64
50
|
end
|
65
51
|
|
52
|
+
def links
|
53
|
+
(href_links + possible_file_links).compact.uniq
|
54
|
+
end
|
55
|
+
|
66
56
|
private
|
67
57
|
|
68
58
|
def response
|
69
59
|
@response ||= get
|
70
60
|
end
|
71
61
|
|
72
|
-
def check(url)
|
73
|
-
res = HTTPClient.head(url)
|
74
|
-
res.status.success?
|
75
|
-
rescue StandardError
|
76
|
-
false
|
77
|
-
end
|
78
|
-
|
79
62
|
def get
|
80
63
|
HTTPClient.get url
|
81
64
|
end
|
@@ -90,12 +73,31 @@ module Miteru
|
|
90
73
|
nil
|
91
74
|
end
|
92
75
|
|
93
|
-
def
|
94
|
-
if doc
|
95
|
-
doc.css("a").map { |a| a.get("href") }.compact
|
76
|
+
def href_links
|
77
|
+
if doc && ok? && index?
|
78
|
+
doc.css("a").map { |a| a.get("href") }.compact.map do |href|
|
79
|
+
href = href.start_with?("/") ? href : "/#{href}"
|
80
|
+
url + href
|
81
|
+
end
|
96
82
|
else
|
97
83
|
[]
|
98
84
|
end
|
85
|
+
rescue Addressable::URI::InvalidURIError, ArgumentError, Encoding::CompatibilityError, HTTP::Error, LL::ParserError, OpenSSL::SSL::SSLError => _e
|
86
|
+
[]
|
87
|
+
end
|
88
|
+
|
89
|
+
def possible_file_links
|
90
|
+
uri = URI.parse(url)
|
91
|
+
|
92
|
+
segments = uri.path.split("/")
|
93
|
+
return [] if segments.length.zero?
|
94
|
+
|
95
|
+
last = segments.last
|
96
|
+
VALID_EXTENSIONS.map do |ext|
|
97
|
+
new_segments = segments[0..-2] + ["#{last}#{ext}"]
|
98
|
+
uri.path = new_segments.join("/")
|
99
|
+
uri.to_s
|
100
|
+
end
|
99
101
|
end
|
100
102
|
end
|
101
103
|
end
|
data/miteru.gemspec
CHANGED
@@ -30,14 +30,14 @@ Gem::Specification.new do |spec|
|
|
30
30
|
spec.add_development_dependency "rake", "~> 13.0"
|
31
31
|
spec.add_development_dependency "rspec", "~> 3.9"
|
32
32
|
spec.add_development_dependency "vcr", "~> 6.0"
|
33
|
-
spec.add_development_dependency "webmock", "~> 3.
|
33
|
+
spec.add_development_dependency "webmock", "~> 3.9"
|
34
34
|
|
35
35
|
spec.add_dependency "colorize", "~> 0.8"
|
36
|
-
spec.add_dependency "down", "~> 5.
|
36
|
+
spec.add_dependency "down", "~> 5.2"
|
37
37
|
spec.add_dependency "http", "~> 4.4"
|
38
|
-
spec.add_dependency "oga", "~> 3.
|
38
|
+
spec.add_dependency "oga", "~> 3.3"
|
39
39
|
spec.add_dependency "parallel", "~> 1.19"
|
40
40
|
spec.add_dependency "slack-notifier", "~> 2.3"
|
41
41
|
spec.add_dependency "thor", "~> 1.0"
|
42
|
-
spec.add_dependency "urlscan", "~> 0.
|
42
|
+
spec.add_dependency "urlscan", "~> 0.6"
|
43
43
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: miteru
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.14.
|
4
|
+
version: 0.14.7
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Manabu Niseki
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-10-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -100,14 +100,14 @@ dependencies:
|
|
100
100
|
requirements:
|
101
101
|
- - "~>"
|
102
102
|
- !ruby/object:Gem::Version
|
103
|
-
version: '3.
|
103
|
+
version: '3.9'
|
104
104
|
type: :development
|
105
105
|
prerelease: false
|
106
106
|
version_requirements: !ruby/object:Gem::Requirement
|
107
107
|
requirements:
|
108
108
|
- - "~>"
|
109
109
|
- !ruby/object:Gem::Version
|
110
|
-
version: '3.
|
110
|
+
version: '3.9'
|
111
111
|
- !ruby/object:Gem::Dependency
|
112
112
|
name: colorize
|
113
113
|
requirement: !ruby/object:Gem::Requirement
|
@@ -128,14 +128,14 @@ dependencies:
|
|
128
128
|
requirements:
|
129
129
|
- - "~>"
|
130
130
|
- !ruby/object:Gem::Version
|
131
|
-
version: '5.
|
131
|
+
version: '5.2'
|
132
132
|
type: :runtime
|
133
133
|
prerelease: false
|
134
134
|
version_requirements: !ruby/object:Gem::Requirement
|
135
135
|
requirements:
|
136
136
|
- - "~>"
|
137
137
|
- !ruby/object:Gem::Version
|
138
|
-
version: '5.
|
138
|
+
version: '5.2'
|
139
139
|
- !ruby/object:Gem::Dependency
|
140
140
|
name: http
|
141
141
|
requirement: !ruby/object:Gem::Requirement
|
@@ -156,14 +156,14 @@ dependencies:
|
|
156
156
|
requirements:
|
157
157
|
- - "~>"
|
158
158
|
- !ruby/object:Gem::Version
|
159
|
-
version: '3.
|
159
|
+
version: '3.3'
|
160
160
|
type: :runtime
|
161
161
|
prerelease: false
|
162
162
|
version_requirements: !ruby/object:Gem::Requirement
|
163
163
|
requirements:
|
164
164
|
- - "~>"
|
165
165
|
- !ruby/object:Gem::Version
|
166
|
-
version: '3.
|
166
|
+
version: '3.3'
|
167
167
|
- !ruby/object:Gem::Dependency
|
168
168
|
name: parallel
|
169
169
|
requirement: !ruby/object:Gem::Requirement
|
@@ -212,14 +212,14 @@ dependencies:
|
|
212
212
|
requirements:
|
213
213
|
- - "~>"
|
214
214
|
- !ruby/object:Gem::Version
|
215
|
-
version: '0.
|
215
|
+
version: '0.6'
|
216
216
|
type: :runtime
|
217
217
|
prerelease: false
|
218
218
|
version_requirements: !ruby/object:Gem::Requirement
|
219
219
|
requirements:
|
220
220
|
- - "~>"
|
221
221
|
- !ruby/object:Gem::Version
|
222
|
-
version: '0.
|
222
|
+
version: '0.6'
|
223
223
|
description: An experimental phishing kit detector
|
224
224
|
email:
|
225
225
|
- manabu.niseki@gmail.com
|
@@ -259,12 +259,13 @@ files:
|
|
259
259
|
- lib/miteru/version.rb
|
260
260
|
- lib/miteru/website.rb
|
261
261
|
- miteru.gemspec
|
262
|
+
- renovate.json
|
262
263
|
- screenshots/slack.png
|
263
264
|
homepage: https://github.com/ninoseki/miteru
|
264
265
|
licenses:
|
265
266
|
- MIT
|
266
267
|
metadata: {}
|
267
|
-
post_install_message:
|
268
|
+
post_install_message:
|
268
269
|
rdoc_options: []
|
269
270
|
require_paths:
|
270
271
|
- lib
|
@@ -280,7 +281,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
280
281
|
version: '0'
|
281
282
|
requirements: []
|
282
283
|
rubygems_version: 3.1.2
|
283
|
-
signing_key:
|
284
|
+
signing_key:
|
284
285
|
specification_version: 4
|
285
286
|
summary: An experimental phishing kit detector
|
286
287
|
test_files: []
|