miteru 0.13.0 → 0.14.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 16f41a7076d910face109e154bbc1a90b3d5a5e1e964be7236b8c798939dee7b
4
- data.tar.gz: d16c59e0e78897c96d78a2e51f84903449c53355eb2436b4e700ba7eb4a07887
3
+ metadata.gz: 89102143e9e405545e94b50da7b81b90c85ae9612c8c914d4b50b6d09ca2e023
4
+ data.tar.gz: 2d6e60e6301425b9ceb8ed7cf103caab144f2ddab4fdd3872ef3f332a6667e37
5
5
  SHA512:
6
- metadata.gz: e83f7f493b2f1015ccfed05b8b1acde61fc093a3e75f29dcb6a1be8f20203f10ccbddbfd4f770277821821bf0ec31eab4e1bcf5587d77c435510a3b583302fc9
7
- data.tar.gz: c9b68154de357f2563fc80ca0059e476e1c919cffa6b67ff4d1a2658d317b6ce7277c168717a26be2b020716da94c0b751dc1e9a6ebac085dc3e2729847c7b0e
6
+ metadata.gz: 3df145d910400efc069d07c718943560a064b5d9fff849fb695795bf49b1d30681e6c31946172f2cd0e7508ca46ded7ff93d2935e48fc6dc970e1817d6e6348c
7
+ data.tar.gz: 70057c7c2451f4631bfda279431e40dfc86bc4931caa555141e607d0a2d73e5308bd0458ddc9046b1e0ccbe0329829ab8568202e8671c68146e5b6b4b6688705
@@ -3,4 +3,5 @@ language: ruby
3
3
  cache: bundler
4
4
  rvm:
5
5
  - 2.6
6
+ - 2.7
6
7
  before_install: gem install bundler -v 2.1
data/README.md CHANGED
@@ -17,6 +17,8 @@ Miteru is an experimental phishing kit detection tool.
17
17
  - [URLhaus feed via urlscan.io](https://urlscan.io/search/#URLHaus)
18
18
  - urlscan.io phish feed (available for Pro users)
19
19
  - [Ayashige feed](https://github.com/ninoseki/ayashige)
20
+ - [Phishing Database feed](https://github.com/mitchellkrogza/Phishing.Database)
21
+ - [PhishStats feed](https://phishstats.info/)
20
22
  - It checks each phishy URL whether it enables directory listing and contains a phishing kit (compressed file) or not.
21
23
  - Note: compressed file = `*.zip`, `*.rar`, `*.7z`, `*.tar` and `*.gz`.
22
24
 
@@ -28,6 +28,8 @@ module Miteru
28
28
  # @return [Boolean]
29
29
  attr_accessor :verbose
30
30
 
31
+ attr_reader :valid_extensions
32
+
31
33
  def initialize
32
34
  @auto_download = false
33
35
  @ayashige = false
@@ -37,6 +39,8 @@ module Miteru
37
39
  @size = 100
38
40
  @threads = Parallel.processor_count
39
41
  @verbose = false
42
+
43
+ @valid_extensions = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
40
44
  end
41
45
 
42
46
  def auto_download?
@@ -1,6 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require_relative "./feeds/feed"
4
+ require_relative "./feeds/phishing_database"
5
+ require_relative "./feeds/phishstats"
4
6
  require_relative "./feeds/ayashige"
5
7
  require_relative "./feeds/urlscan"
6
8
  require_relative "./feeds/urlscan_pro"
@@ -11,6 +13,8 @@ module Miteru
11
13
 
12
14
  def initialize
13
15
  @feeds = [
16
+ PhishingDatabase.new,
17
+ PhishStats.new,
14
18
  UrlScan.new(Miteru.configuration.size),
15
19
  UrlScanPro.new,
16
20
  Miteru.configuration.ayashige? ? Ayashige.new : nil
@@ -45,6 +49,7 @@ module Miteru
45
49
  return [base] if segments.length.zero?
46
50
 
47
51
  urls = (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join('/')}" }
52
+
48
53
  urls.reject do |breakdowned_url|
49
54
  # Reject a url which ends with specific extension names
50
55
  invalid_extension? breakdowned_url
@@ -0,0 +1,20 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+ require "uri"
5
+
6
+ module Miteru
7
+ class Feeds
8
+ class PhishingDatabase < Feed
9
+ URL = "https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/phishing-links-NEW-today.txt"
10
+
11
+ def urls
12
+ body = get(URL)
13
+ body.to_s.lines.map(&:chomp)
14
+ rescue HTTPResponseError, HTTP::Error, JSON::ParserError => e
15
+ puts "Failed to load phishing database feed (#{e})"
16
+ []
17
+ end
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,28 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+ require "uri"
5
+
6
+ module Miteru
7
+ class Feeds
8
+ class PhishStats < Feed
9
+ URL = "https://phishstats.info:2096/api/phishing?_sort=-id&size=100"
10
+
11
+ def urls
12
+ json = JSON.parse(get(URL))
13
+ json.map do |entry|
14
+ entry.dig("url")
15
+ end
16
+ rescue HTTPResponseError, HTTP::Error, JSON::ParserError => e
17
+ puts "Failed to load PhishStats feed (#{e})"
18
+ []
19
+ end
20
+
21
+ private
22
+
23
+ def url_for(path)
24
+ URI(URL + path)
25
+ end
26
+ end
27
+ end
28
+ end
@@ -18,11 +18,20 @@ module Miteru
18
18
  end
19
19
 
20
20
  def download(url, destination)
21
- down = Down::Http.new(default_options) { |client| client.headers(default_headers) }
21
+ down = Down::Http.new(**default_options) { |client| client.headers(**default_headers) }
22
22
  down.download(url, destination: destination)
23
23
  destination
24
24
  end
25
25
 
26
+ def head(url, options = {})
27
+ options = options.merge default_options
28
+
29
+ HTTP.follow
30
+ .timeout(3)
31
+ .headers(urlscan_url?(url) ? urlscan_headers : default_headers)
32
+ .head(url, options)
33
+ end
34
+
26
35
  def get(url, options = {})
27
36
  options = options.merge default_options
28
37
 
@@ -48,6 +57,10 @@ module Miteru
48
57
  def post(url, options = {})
49
58
  new.post url, options
50
59
  end
60
+
61
+ def head(url, options = {})
62
+ new.head url, options
63
+ end
51
64
  end
52
65
 
53
66
  private
@@ -5,37 +5,32 @@ require "securerandom"
5
5
 
6
6
  module Miteru
7
7
  class Kit
8
- VALID_EXTENSIONS = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
8
+ VALID_EXTENSIONS = Miteru.configuration.valid_extensions
9
9
 
10
- attr_reader :base_url, :link
10
+ attr_reader :url
11
11
 
12
- def initialize(base_url:, link:)
13
- @base_url = base_url
14
- @link = link.start_with?("/") ? link[1..-1] : link
12
+ def initialize(url)
13
+ @url = url
15
14
  end
16
15
 
17
- def valid?
18
- VALID_EXTENSIONS.include? extname
16
+ def valid?;
17
+ valid_ext? && reachable_and_archive_file?
19
18
  end
20
19
 
21
20
  def extname
22
- return ".tar.gz" if link.end_with?("tar.gz")
21
+ return ".tar.gz" if url.end_with?("tar.gz")
23
22
 
24
- File.extname(link)
23
+ File.extname(url)
25
24
  end
26
25
 
27
26
  def basename
28
- File.basename(link)
27
+ File.basename(url)
29
28
  end
30
29
 
31
30
  def filename
32
31
  CGI.unescape basename
33
32
  end
34
33
 
35
- def url
36
- "#{base_url}/#{basename}"
37
- end
38
-
39
34
  def download_filepath
40
35
  "#{base_dir}/#{download_filename}"
41
36
  end
@@ -59,7 +54,7 @@ module Miteru
59
54
  end
60
55
 
61
56
  def hostname
62
- URI(base_url).hostname
57
+ URI(url).hostname
63
58
  end
64
59
 
65
60
  def download_filename
@@ -69,5 +64,24 @@ module Miteru
69
64
  def base_dir
70
65
  @base_dir ||= Miteru.configuration.download_to
71
66
  end
67
+
68
+ def valid_ext?
69
+ VALID_EXTENSIONS.include? extname
70
+ end
71
+
72
+ def reachable?(response)
73
+ response.status.success?
74
+ end
75
+
76
+ def archive_file?(response)
77
+ !response.content_type.mime_type.to_s.start_with? "text/"
78
+ end
79
+
80
+ def reachable_and_archive_file?
81
+ res = HTTPClient.head(url)
82
+ reachable?(res) && archive_file?(res)
83
+ rescue StandardError
84
+ false
85
+ end
72
86
  end
73
87
  end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Miteru
4
- VERSION = "0.13.0"
4
+ VERSION = "0.14.4"
5
5
  end
@@ -4,7 +4,10 @@ require "oga"
4
4
 
5
5
  module Miteru
6
6
  class Website
7
+ VALID_EXTENSIONS = Miteru.configuration.valid_extensions
8
+
7
9
  attr_reader :url
10
+
8
11
  def initialize(url)
9
12
  @url = url
10
13
  end
@@ -15,7 +18,7 @@ module Miteru
15
18
 
16
19
  def kits
17
20
  @kits ||= links.map do |link|
18
- kit = Kit.new(base_url: url, link: link.to_s)
21
+ kit = Kit.new(link)
19
22
  kit.valid? ? kit : nil
20
23
  end.compact
21
24
  end
@@ -33,7 +36,7 @@ module Miteru
33
36
  end
34
37
 
35
38
  def has_kits?
36
- ok? && index? && kits?
39
+ kits?
37
40
  rescue Addressable::URI::InvalidURIError, ArgumentError, Encoding::CompatibilityError, HTTP::Error, LL::ParserError, OpenSSL::SSL::SSLError => _e
38
41
  false
39
42
  end
@@ -46,6 +49,10 @@ module Miteru
46
49
  "It might contain #{noun}: #{filename_with_sizes}."
47
50
  end
48
51
 
52
+ def links
53
+ (href_links + possible_file_links).compact.uniq
54
+ end
55
+
49
56
  private
50
57
 
51
58
  def response
@@ -66,12 +73,31 @@ module Miteru
66
73
  nil
67
74
  end
68
75
 
69
- def links
70
- if doc
71
- doc.css("a").map { |a| a.get("href") }.compact
76
+ def href_links
77
+ if doc && ok? && index?
78
+ doc.css("a").map { |a| a.get("href") }.compact.map do |href|
79
+ href = href.start_with?("/") ? href : "/#{href}"
80
+ url + href
81
+ end
72
82
  else
73
83
  []
74
84
  end
85
+ rescue Addressable::URI::InvalidURIError, ArgumentError, Encoding::CompatibilityError, HTTP::Error, LL::ParserError, OpenSSL::SSL::SSLError => _e
86
+ []
87
+ end
88
+
89
+ def possible_file_links
90
+ uri = URI.parse(url)
91
+
92
+ segments = uri.path.split("/")
93
+ return [] if segments.length.zero?
94
+
95
+ last = segments.last
96
+ VALID_EXTENSIONS.map do |ext|
97
+ new_segments = segments[0..-2] + ["#{last}#{ext}"]
98
+ uri.path = new_segments.join("/")
99
+ uri.to_s
100
+ end
75
101
  end
76
102
  end
77
103
  end
@@ -24,18 +24,18 @@ Gem::Specification.new do |spec|
24
24
  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
25
25
  spec.require_paths = ["lib"]
26
26
 
27
- spec.add_development_dependency "bundler", "~> 2.0"
27
+ spec.add_development_dependency "bundler", "~> 2.1"
28
28
  spec.add_development_dependency "coveralls", "~> 0.8"
29
29
  spec.add_development_dependency "glint", "~> 0.1"
30
30
  spec.add_development_dependency "rake", "~> 13.0"
31
31
  spec.add_development_dependency "rspec", "~> 3.9"
32
- spec.add_development_dependency "vcr", "~> 5.0"
33
- spec.add_development_dependency "webmock", "~> 3.7"
32
+ spec.add_development_dependency "vcr", "~> 6.0"
33
+ spec.add_development_dependency "webmock", "~> 3.8"
34
34
 
35
35
  spec.add_dependency "colorize", "~> 0.8"
36
- spec.add_dependency "down", "~> 5.0"
37
- spec.add_dependency "http", "~> 4.2"
38
- spec.add_dependency "oga", "~> 3.0"
36
+ spec.add_dependency "down", "~> 5.1"
37
+ spec.add_dependency "http", "~> 4.4"
38
+ spec.add_dependency "oga", "~> 3.2"
39
39
  spec.add_dependency "parallel", "~> 1.19"
40
40
  spec.add_dependency "slack-notifier", "~> 2.3"
41
41
  spec.add_dependency "thor", "~> 1.0"
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: miteru
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.13.0
4
+ version: 0.14.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Manabu Niseki
8
- autorequire:
8
+ autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2019-12-17 00:00:00.000000000 Z
11
+ date: 2020-06-30 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -16,14 +16,14 @@ dependencies:
16
16
  requirements:
17
17
  - - "~>"
18
18
  - !ruby/object:Gem::Version
19
- version: '2.0'
19
+ version: '2.1'
20
20
  type: :development
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - "~>"
25
25
  - !ruby/object:Gem::Version
26
- version: '2.0'
26
+ version: '2.1'
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: coveralls
29
29
  requirement: !ruby/object:Gem::Requirement
@@ -86,28 +86,28 @@ dependencies:
86
86
  requirements:
87
87
  - - "~>"
88
88
  - !ruby/object:Gem::Version
89
- version: '5.0'
89
+ version: '6.0'
90
90
  type: :development
91
91
  prerelease: false
92
92
  version_requirements: !ruby/object:Gem::Requirement
93
93
  requirements:
94
94
  - - "~>"
95
95
  - !ruby/object:Gem::Version
96
- version: '5.0'
96
+ version: '6.0'
97
97
  - !ruby/object:Gem::Dependency
98
98
  name: webmock
99
99
  requirement: !ruby/object:Gem::Requirement
100
100
  requirements:
101
101
  - - "~>"
102
102
  - !ruby/object:Gem::Version
103
- version: '3.7'
103
+ version: '3.8'
104
104
  type: :development
105
105
  prerelease: false
106
106
  version_requirements: !ruby/object:Gem::Requirement
107
107
  requirements:
108
108
  - - "~>"
109
109
  - !ruby/object:Gem::Version
110
- version: '3.7'
110
+ version: '3.8'
111
111
  - !ruby/object:Gem::Dependency
112
112
  name: colorize
113
113
  requirement: !ruby/object:Gem::Requirement
@@ -128,42 +128,42 @@ dependencies:
128
128
  requirements:
129
129
  - - "~>"
130
130
  - !ruby/object:Gem::Version
131
- version: '5.0'
131
+ version: '5.1'
132
132
  type: :runtime
133
133
  prerelease: false
134
134
  version_requirements: !ruby/object:Gem::Requirement
135
135
  requirements:
136
136
  - - "~>"
137
137
  - !ruby/object:Gem::Version
138
- version: '5.0'
138
+ version: '5.1'
139
139
  - !ruby/object:Gem::Dependency
140
140
  name: http
141
141
  requirement: !ruby/object:Gem::Requirement
142
142
  requirements:
143
143
  - - "~>"
144
144
  - !ruby/object:Gem::Version
145
- version: '4.2'
145
+ version: '4.4'
146
146
  type: :runtime
147
147
  prerelease: false
148
148
  version_requirements: !ruby/object:Gem::Requirement
149
149
  requirements:
150
150
  - - "~>"
151
151
  - !ruby/object:Gem::Version
152
- version: '4.2'
152
+ version: '4.4'
153
153
  - !ruby/object:Gem::Dependency
154
154
  name: oga
155
155
  requirement: !ruby/object:Gem::Requirement
156
156
  requirements:
157
157
  - - "~>"
158
158
  - !ruby/object:Gem::Version
159
- version: '3.0'
159
+ version: '3.2'
160
160
  type: :runtime
161
161
  prerelease: false
162
162
  version_requirements: !ruby/object:Gem::Requirement
163
163
  requirements:
164
164
  - - "~>"
165
165
  - !ruby/object:Gem::Version
166
- version: '3.0'
166
+ version: '3.2'
167
167
  - !ruby/object:Gem::Dependency
168
168
  name: parallel
169
169
  requirement: !ruby/object:Gem::Requirement
@@ -249,6 +249,8 @@ files:
249
249
  - lib/miteru/feeds.rb
250
250
  - lib/miteru/feeds/ayashige.rb
251
251
  - lib/miteru/feeds/feed.rb
252
+ - lib/miteru/feeds/phishing_database.rb
253
+ - lib/miteru/feeds/phishstats.rb
252
254
  - lib/miteru/feeds/urlscan.rb
253
255
  - lib/miteru/feeds/urlscan_pro.rb
254
256
  - lib/miteru/http_client.rb
@@ -262,7 +264,7 @@ homepage: https://github.com/ninoseki/miteru
262
264
  licenses:
263
265
  - MIT
264
266
  metadata: {}
265
- post_install_message:
267
+ post_install_message:
266
268
  rdoc_options: []
267
269
  require_paths:
268
270
  - lib
@@ -277,8 +279,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
277
279
  - !ruby/object:Gem::Version
278
280
  version: '0'
279
281
  requirements: []
280
- rubygems_version: 3.0.3
281
- signing_key:
282
+ rubygems_version: 3.1.2
283
+ signing_key:
282
284
  specification_version: 4
283
285
  summary: An experimental phishing kit detector
284
286
  test_files: []