miteru 0.13.0 → 0.14.4

This diff has not been reviewed by any users.
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 16f41a7076d910face109e154bbc1a90b3d5a5e1e964be7236b8c798939dee7b
4
- data.tar.gz: d16c59e0e78897c96d78a2e51f84903449c53355eb2436b4e700ba7eb4a07887
3
+ metadata.gz: 89102143e9e405545e94b50da7b81b90c85ae9612c8c914d4b50b6d09ca2e023
4
+ data.tar.gz: 2d6e60e6301425b9ceb8ed7cf103caab144f2ddab4fdd3872ef3f332a6667e37
5
5
  SHA512:
6
- metadata.gz: e83f7f493b2f1015ccfed05b8b1acde61fc093a3e75f29dcb6a1be8f20203f10ccbddbfd4f770277821821bf0ec31eab4e1bcf5587d77c435510a3b583302fc9
7
- data.tar.gz: c9b68154de357f2563fc80ca0059e476e1c919cffa6b67ff4d1a2658d317b6ce7277c168717a26be2b020716da94c0b751dc1e9a6ebac085dc3e2729847c7b0e
6
+ metadata.gz: 3df145d910400efc069d07c718943560a064b5d9fff849fb695795bf49b1d30681e6c31946172f2cd0e7508ca46ded7ff93d2935e48fc6dc970e1817d6e6348c
7
+ data.tar.gz: 70057c7c2451f4631bfda279431e40dfc86bc4931caa555141e607d0a2d73e5308bd0458ddc9046b1e0ccbe0329829ab8568202e8671c68146e5b6b4b6688705
@@ -3,4 +3,5 @@ language: ruby
3
3
  cache: bundler
4
4
  rvm:
5
5
  - 2.6
6
+ - 2.7
6
7
  before_install: gem install bundler -v 2.1
data/README.md CHANGED
@@ -17,6 +17,8 @@ Miteru is an experimental phishing kit detection tool.
17
17
  - [URLhaus feed via urlscan.io](https://urlscan.io/search/#URLHaus)
18
18
  - urlscan.io phish feed (available for Pro users)
19
19
  - [Ayashige feed](https://github.com/ninoseki/ayashige)
20
+ - [Phishing Database feed](https://github.com/mitchellkrogza/Phishing.Database)
21
+ - [PhishStats feed](https://phishstats.info/)
20
22
  - It checks each phishy URL whether it enables directory listing and contains a phishing kit (compressed file) or not.
21
23
  - Note: compressed file = `*.zip`, `*.rar`, `*.7z`, `*.tar` and `*.gz`.
22
24
 
@@ -28,6 +28,8 @@ module Miteru
28
28
  # @return [Boolean]
29
29
  attr_accessor :verbose
30
30
 
31
+ attr_reader :valid_extensions
32
+
31
33
  def initialize
32
34
  @auto_download = false
33
35
  @ayashige = false
@@ -37,6 +39,8 @@ module Miteru
37
39
  @size = 100
38
40
  @threads = Parallel.processor_count
39
41
  @verbose = false
42
+
43
+ @valid_extensions = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
40
44
  end
41
45
 
42
46
  def auto_download?
@@ -1,6 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require_relative "./feeds/feed"
4
+ require_relative "./feeds/phishing_database"
5
+ require_relative "./feeds/phishstats"
4
6
  require_relative "./feeds/ayashige"
5
7
  require_relative "./feeds/urlscan"
6
8
  require_relative "./feeds/urlscan_pro"
@@ -11,6 +13,8 @@ module Miteru
11
13
 
12
14
  def initialize
13
15
  @feeds = [
16
+ PhishingDatabase.new,
17
+ PhishStats.new,
14
18
  UrlScan.new(Miteru.configuration.size),
15
19
  UrlScanPro.new,
16
20
  Miteru.configuration.ayashige? ? Ayashige.new : nil
@@ -45,6 +49,7 @@ module Miteru
45
49
  return [base] if segments.length.zero?
46
50
 
47
51
  urls = (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join('/')}" }
52
+
48
53
  urls.reject do |breakdowned_url|
49
54
  # Reject a url which ends with specific extension names
50
55
  invalid_extension? breakdowned_url
@@ -0,0 +1,20 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+ require "uri"
5
+
6
+ module Miteru
7
+ class Feeds
8
+ class PhishingDatabase < Feed
9
+ URL = "https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/phishing-links-NEW-today.txt"
10
+
11
+ def urls
12
+ body = get(URL)
13
+ body.to_s.lines.map(&:chomp)
14
+ rescue HTTPResponseError, HTTP::Error, JSON::ParserError => e
15
+ puts "Failed to load phishing database feed (#{e})"
16
+ []
17
+ end
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,28 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+ require "uri"
5
+
6
+ module Miteru
7
+ class Feeds
8
+ class PhishStats < Feed
9
+ URL = "https://phishstats.info:2096/api/phishing?_sort=-id&size=100"
10
+
11
+ def urls
12
+ json = JSON.parse(get(URL))
13
+ json.map do |entry|
14
+ entry.dig("url")
15
+ end
16
+ rescue HTTPResponseError, HTTP::Error, JSON::ParserError => e
17
+ puts "Failed to load PhishStats feed (#{e})"
18
+ []
19
+ end
20
+
21
+ private
22
+
23
+ def url_for(path)
24
+ URI(URL + path)
25
+ end
26
+ end
27
+ end
28
+ end
@@ -18,11 +18,20 @@ module Miteru
18
18
  end
19
19
 
20
20
  def download(url, destination)
21
- down = Down::Http.new(default_options) { |client| client.headers(default_headers) }
21
+ down = Down::Http.new(**default_options) { |client| client.headers(**default_headers) }
22
22
  down.download(url, destination: destination)
23
23
  destination
24
24
  end
25
25
 
26
+ def head(url, options = {})
27
+ options = options.merge default_options
28
+
29
+ HTTP.follow
30
+ .timeout(3)
31
+ .headers(urlscan_url?(url) ? urlscan_headers : default_headers)
32
+ .head(url, options)
33
+ end
34
+
26
35
  def get(url, options = {})
27
36
  options = options.merge default_options
28
37
 
@@ -48,6 +57,10 @@ module Miteru
48
57
  def post(url, options = {})
49
58
  new.post url, options
50
59
  end
60
+
61
+ def head(url, options = {})
62
+ new.head url, options
63
+ end
51
64
  end
52
65
 
53
66
  private
@@ -5,37 +5,32 @@ require "securerandom"
5
5
 
6
6
  module Miteru
7
7
  class Kit
8
- VALID_EXTENSIONS = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
8
+ VALID_EXTENSIONS = Miteru.configuration.valid_extensions
9
9
 
10
- attr_reader :base_url, :link
10
+ attr_reader :url
11
11
 
12
- def initialize(base_url:, link:)
13
- @base_url = base_url
14
- @link = link.start_with?("/") ? link[1..-1] : link
12
+ def initialize(url)
13
+ @url = url
15
14
  end
16
15
 
17
- def valid?
18
- VALID_EXTENSIONS.include? extname
16
+ def valid?;
17
+ valid_ext? && reachable_and_archive_file?
19
18
  end
20
19
 
21
20
  def extname
22
- return ".tar.gz" if link.end_with?("tar.gz")
21
+ return ".tar.gz" if url.end_with?("tar.gz")
23
22
 
24
- File.extname(link)
23
+ File.extname(url)
25
24
  end
26
25
 
27
26
  def basename
28
- File.basename(link)
27
+ File.basename(url)
29
28
  end
30
29
 
31
30
  def filename
32
31
  CGI.unescape basename
33
32
  end
34
33
 
35
- def url
36
- "#{base_url}/#{basename}"
37
- end
38
-
39
34
  def download_filepath
40
35
  "#{base_dir}/#{download_filename}"
41
36
  end
@@ -59,7 +54,7 @@ module Miteru
59
54
  end
60
55
 
61
56
  def hostname
62
- URI(base_url).hostname
57
+ URI(url).hostname
63
58
  end
64
59
 
65
60
  def download_filename
@@ -69,5 +64,24 @@ module Miteru
69
64
  def base_dir
70
65
  @base_dir ||= Miteru.configuration.download_to
71
66
  end
67
+
68
+ def valid_ext?
69
+ VALID_EXTENSIONS.include? extname
70
+ end
71
+
72
+ def reachable?(response)
73
+ response.status.success?
74
+ end
75
+
76
+ def archive_file?(response)
77
+ !response.content_type.mime_type.to_s.start_with? "text/"
78
+ end
79
+
80
+ def reachable_and_archive_file?
81
+ res = HTTPClient.head(url)
82
+ reachable?(res) && archive_file?(res)
83
+ rescue StandardError
84
+ false
85
+ end
72
86
  end
73
87
  end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Miteru
4
- VERSION = "0.13.0"
4
+ VERSION = "0.14.4"
5
5
  end
@@ -4,7 +4,10 @@ require "oga"
4
4
 
5
5
  module Miteru
6
6
  class Website
7
+ VALID_EXTENSIONS = Miteru.configuration.valid_extensions
8
+
7
9
  attr_reader :url
10
+
8
11
  def initialize(url)
9
12
  @url = url
10
13
  end
@@ -15,7 +18,7 @@ module Miteru
15
18
 
16
19
  def kits
17
20
  @kits ||= links.map do |link|
18
- kit = Kit.new(base_url: url, link: link.to_s)
21
+ kit = Kit.new(link)
19
22
  kit.valid? ? kit : nil
20
23
  end.compact
21
24
  end
@@ -33,7 +36,7 @@ module Miteru
33
36
  end
34
37
 
35
38
  def has_kits?
36
- ok? && index? && kits?
39
+ kits?
37
40
  rescue Addressable::URI::InvalidURIError, ArgumentError, Encoding::CompatibilityError, HTTP::Error, LL::ParserError, OpenSSL::SSL::SSLError => _e
38
41
  false
39
42
  end
@@ -46,6 +49,10 @@ module Miteru
46
49
  "It might contain #{noun}: #{filename_with_sizes}."
47
50
  end
48
51
 
52
+ def links
53
+ (href_links + possible_file_links).compact.uniq
54
+ end
55
+
49
56
  private
50
57
 
51
58
  def response
@@ -66,12 +73,31 @@ module Miteru
66
73
  nil
67
74
  end
68
75
 
69
- def links
70
- if doc
71
- doc.css("a").map { |a| a.get("href") }.compact
76
+ def href_links
77
+ if doc && ok? && index?
78
+ doc.css("a").map { |a| a.get("href") }.compact.map do |href|
79
+ href = href.start_with?("/") ? href : "/#{href}"
80
+ url + href
81
+ end
72
82
  else
73
83
  []
74
84
  end
85
+ rescue Addressable::URI::InvalidURIError, ArgumentError, Encoding::CompatibilityError, HTTP::Error, LL::ParserError, OpenSSL::SSL::SSLError => _e
86
+ []
87
+ end
88
+
89
+ def possible_file_links
90
+ uri = URI.parse(url)
91
+
92
+ segments = uri.path.split("/")
93
+ return [] if segments.length.zero?
94
+
95
+ last = segments.last
96
+ VALID_EXTENSIONS.map do |ext|
97
+ new_segments = segments[0..-2] + ["#{last}#{ext}"]
98
+ uri.path = new_segments.join("/")
99
+ uri.to_s
100
+ end
75
101
  end
76
102
  end
77
103
  end
@@ -24,18 +24,18 @@ Gem::Specification.new do |spec|
24
24
  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
25
25
  spec.require_paths = ["lib"]
26
26
 
27
- spec.add_development_dependency "bundler", "~> 2.0"
27
+ spec.add_development_dependency "bundler", "~> 2.1"
28
28
  spec.add_development_dependency "coveralls", "~> 0.8"
29
29
  spec.add_development_dependency "glint", "~> 0.1"
30
30
  spec.add_development_dependency "rake", "~> 13.0"
31
31
  spec.add_development_dependency "rspec", "~> 3.9"
32
- spec.add_development_dependency "vcr", "~> 5.0"
33
- spec.add_development_dependency "webmock", "~> 3.7"
32
+ spec.add_development_dependency "vcr", "~> 6.0"
33
+ spec.add_development_dependency "webmock", "~> 3.8"
34
34
 
35
35
  spec.add_dependency "colorize", "~> 0.8"
36
- spec.add_dependency "down", "~> 5.0"
37
- spec.add_dependency "http", "~> 4.2"
38
- spec.add_dependency "oga", "~> 3.0"
36
+ spec.add_dependency "down", "~> 5.1"
37
+ spec.add_dependency "http", "~> 4.4"
38
+ spec.add_dependency "oga", "~> 3.2"
39
39
  spec.add_dependency "parallel", "~> 1.19"
40
40
  spec.add_dependency "slack-notifier", "~> 2.3"
41
41
  spec.add_dependency "thor", "~> 1.0"
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: miteru
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.13.0
4
+ version: 0.14.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Manabu Niseki
8
- autorequire:
8
+ autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2019-12-17 00:00:00.000000000 Z
11
+ date: 2020-06-30 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -16,14 +16,14 @@ dependencies:
16
16
  requirements:
17
17
  - - "~>"
18
18
  - !ruby/object:Gem::Version
19
- version: '2.0'
19
+ version: '2.1'
20
20
  type: :development
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - "~>"
25
25
  - !ruby/object:Gem::Version
26
- version: '2.0'
26
+ version: '2.1'
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: coveralls
29
29
  requirement: !ruby/object:Gem::Requirement
@@ -86,28 +86,28 @@ dependencies:
86
86
  requirements:
87
87
  - - "~>"
88
88
  - !ruby/object:Gem::Version
89
- version: '5.0'
89
+ version: '6.0'
90
90
  type: :development
91
91
  prerelease: false
92
92
  version_requirements: !ruby/object:Gem::Requirement
93
93
  requirements:
94
94
  - - "~>"
95
95
  - !ruby/object:Gem::Version
96
- version: '5.0'
96
+ version: '6.0'
97
97
  - !ruby/object:Gem::Dependency
98
98
  name: webmock
99
99
  requirement: !ruby/object:Gem::Requirement
100
100
  requirements:
101
101
  - - "~>"
102
102
  - !ruby/object:Gem::Version
103
- version: '3.7'
103
+ version: '3.8'
104
104
  type: :development
105
105
  prerelease: false
106
106
  version_requirements: !ruby/object:Gem::Requirement
107
107
  requirements:
108
108
  - - "~>"
109
109
  - !ruby/object:Gem::Version
110
- version: '3.7'
110
+ version: '3.8'
111
111
  - !ruby/object:Gem::Dependency
112
112
  name: colorize
113
113
  requirement: !ruby/object:Gem::Requirement
@@ -128,42 +128,42 @@ dependencies:
128
128
  requirements:
129
129
  - - "~>"
130
130
  - !ruby/object:Gem::Version
131
- version: '5.0'
131
+ version: '5.1'
132
132
  type: :runtime
133
133
  prerelease: false
134
134
  version_requirements: !ruby/object:Gem::Requirement
135
135
  requirements:
136
136
  - - "~>"
137
137
  - !ruby/object:Gem::Version
138
- version: '5.0'
138
+ version: '5.1'
139
139
  - !ruby/object:Gem::Dependency
140
140
  name: http
141
141
  requirement: !ruby/object:Gem::Requirement
142
142
  requirements:
143
143
  - - "~>"
144
144
  - !ruby/object:Gem::Version
145
- version: '4.2'
145
+ version: '4.4'
146
146
  type: :runtime
147
147
  prerelease: false
148
148
  version_requirements: !ruby/object:Gem::Requirement
149
149
  requirements:
150
150
  - - "~>"
151
151
  - !ruby/object:Gem::Version
152
- version: '4.2'
152
+ version: '4.4'
153
153
  - !ruby/object:Gem::Dependency
154
154
  name: oga
155
155
  requirement: !ruby/object:Gem::Requirement
156
156
  requirements:
157
157
  - - "~>"
158
158
  - !ruby/object:Gem::Version
159
- version: '3.0'
159
+ version: '3.2'
160
160
  type: :runtime
161
161
  prerelease: false
162
162
  version_requirements: !ruby/object:Gem::Requirement
163
163
  requirements:
164
164
  - - "~>"
165
165
  - !ruby/object:Gem::Version
166
- version: '3.0'
166
+ version: '3.2'
167
167
  - !ruby/object:Gem::Dependency
168
168
  name: parallel
169
169
  requirement: !ruby/object:Gem::Requirement
@@ -249,6 +249,8 @@ files:
249
249
  - lib/miteru/feeds.rb
250
250
  - lib/miteru/feeds/ayashige.rb
251
251
  - lib/miteru/feeds/feed.rb
252
+ - lib/miteru/feeds/phishing_database.rb
253
+ - lib/miteru/feeds/phishstats.rb
252
254
  - lib/miteru/feeds/urlscan.rb
253
255
  - lib/miteru/feeds/urlscan_pro.rb
254
256
  - lib/miteru/http_client.rb
@@ -262,7 +264,7 @@ homepage: https://github.com/ninoseki/miteru
262
264
  licenses:
263
265
  - MIT
264
266
  metadata: {}
265
- post_install_message:
267
+ post_install_message:
266
268
  rdoc_options: []
267
269
  require_paths:
268
270
  - lib
@@ -277,8 +279,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
277
279
  - !ruby/object:Gem::Version
278
280
  version: '0'
279
281
  requirements: []
280
- rubygems_version: 3.0.3
281
- signing_key:
282
+ rubygems_version: 3.1.2
283
+ signing_key:
282
284
  specification_version: 4
283
285
  summary: An experimental phishing kit detector
284
286
  test_files: []