miteru 0.14.0 → 0.14.5

This diff has not been reviewed by any users.
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 5bcc07aabe006f3107c410263f09c4580081933b7a7aebfa7344977f6fd05dc6
4
- data.tar.gz: 49cbc8f6efb26eb47ea776c46986bf949f77371a31109e6298bfd11fe7ca670f
3
+ metadata.gz: 1cf6445b9388cdcfb797cdb8c91615a288757508874d0550bab8c6b052c375d6
4
+ data.tar.gz: dd9f428317a0f325c2c467234dda21f05bb7cd616114f49adeec9374a4680451
5
5
  SHA512:
6
- metadata.gz: 54b51311d0f982c2da7221c09384847d6bd526356158dc56948f1d898f3631ef22f3005a041ca0ebc31cfc22bcafc0d64047d95392857f79c8611dd2802bb366
7
- data.tar.gz: c61864c67eb49c79aaa180827610fab24e2fa3de7e14e8396c9ff6ca60439cbe271e313aff6618a8986c2a8eb8d889e018ef062aa92dd2c62afe7ea54bfd7f98
6
+ metadata.gz: 5f740f27cacdb1020a49bc787aa8486cbe6b6ac6fe5e4ad3093af0601928dcd12c78895c39c548a28de18fc35766dbb89507687c81e2bb162d6e69e111242fbd
7
+ data.tar.gz: a110c028b6129a2d0bc903962d1f945be96426b8c909ae6e34bb1955d3ca657b1c9ef5a48bc1e147c54f992517cc86c0ca18047a79cdcb2b2a95bda5c5148caa
@@ -28,6 +28,10 @@ module Miteru
28
28
  # @return [Boolean]
29
29
  attr_accessor :verbose
30
30
 
31
+ attr_reader :valid_extensions
32
+
33
+ attr_reader :valid_mime_types
34
+
31
35
  def initialize
32
36
  @auto_download = false
33
37
  @ayashige = false
@@ -37,6 +41,9 @@ module Miteru
37
41
  @size = 100
38
42
  @threads = Parallel.processor_count
39
43
  @verbose = false
44
+
45
+ @valid_extensions = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
46
+ @valid_mime_types = ["application/zip", "application/vnd.rar", "application/x-7z-compressed", "application/x-tar", "application/gzip"]
40
47
  end
41
48
 
42
49
  def auto_download?
@@ -49,6 +49,7 @@ module Miteru
49
49
  return [base] if segments.length.zero?
50
50
 
51
51
  urls = (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join('/')}" }
52
+
52
53
  urls.reject do |breakdowned_url|
53
54
  # Reject a url which ends with specific extension names
54
55
  invalid_extension? breakdowned_url
@@ -18,11 +18,20 @@ module Miteru
18
18
  end
19
19
 
20
20
  def download(url, destination)
21
- down = Down::Http.new(default_options) { |client| client.headers(default_headers) }
21
+ down = Down::Http.new(**default_options) { |client| client.headers(**default_headers) }
22
22
  down.download(url, destination: destination)
23
23
  destination
24
24
  end
25
25
 
26
+ def head(url, options = {})
27
+ options = options.merge default_options
28
+
29
+ HTTP.follow
30
+ .timeout(3)
31
+ .headers(urlscan_url?(url) ? urlscan_headers : default_headers)
32
+ .head(url, options)
33
+ end
34
+
26
35
  def get(url, options = {})
27
36
  options = options.merge default_options
28
37
 
@@ -48,6 +57,10 @@ module Miteru
48
57
  def post(url, options = {})
49
58
  new.post url, options
50
59
  end
60
+
61
+ def head(url, options = {})
62
+ new.head url, options
63
+ end
51
64
  end
52
65
 
53
66
  private
@@ -5,37 +5,33 @@ require "securerandom"
5
5
 
6
6
  module Miteru
7
7
  class Kit
8
- VALID_EXTENSIONS = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
8
+ VALID_EXTENSIONS = Miteru.configuration.valid_extensions
9
+ VALID_MIME_TYPES = Miteru.configuration.valid_mime_types
9
10
 
10
- attr_reader :base_url, :link
11
+ attr_reader :url
11
12
 
12
- def initialize(base_url:, link:)
13
- @base_url = base_url
14
- @link = link.start_with?("/") ? link[1..-1] : link
13
+ def initialize(url)
14
+ @url = url
15
15
  end
16
16
 
17
- def valid?
18
- VALID_EXTENSIONS.include? extname
17
+ def valid?;
18
+ valid_ext? && reachable_and_valid_mime_type?
19
19
  end
20
20
 
21
21
  def extname
22
- return ".tar.gz" if link.end_with?("tar.gz")
22
+ return ".tar.gz" if url.end_with?("tar.gz")
23
23
 
24
- File.extname(link)
24
+ File.extname(url)
25
25
  end
26
26
 
27
27
  def basename
28
- File.basename(link)
28
+ File.basename(url)
29
29
  end
30
30
 
31
31
  def filename
32
32
  CGI.unescape basename
33
33
  end
34
34
 
35
- def url
36
- "#{base_url}/#{basename}"
37
- end
38
-
39
35
  def download_filepath
40
36
  "#{base_dir}/#{download_filename}"
41
37
  end
@@ -59,7 +55,7 @@ module Miteru
59
55
  end
60
56
 
61
57
  def hostname
62
- URI(base_url).hostname
58
+ URI(url).hostname
63
59
  end
64
60
 
65
61
  def download_filename
@@ -69,5 +65,25 @@ module Miteru
69
65
  def base_dir
70
66
  @base_dir ||= Miteru.configuration.download_to
71
67
  end
68
+
69
+ def valid_ext?
70
+ VALID_EXTENSIONS.include? extname
71
+ end
72
+
73
+ def reachable?(response)
74
+ response.status.success?
75
+ end
76
+
77
+ def valid_mime_type?(response)
78
+ mime_type = response.content_type.mime_type.to_s
79
+ VALID_MIME_TYPES.include? mime_type
80
+ end
81
+
82
+ def reachable_and_valid_mime_type?
83
+ res = HTTPClient.head(url)
84
+ reachable?(res) && valid_mime_type?(res)
85
+ rescue StandardError
86
+ false
87
+ end
72
88
  end
73
89
  end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Miteru
4
- VERSION = "0.14.0"
4
+ VERSION = "0.14.5"
5
5
  end
@@ -4,7 +4,10 @@ require "oga"
4
4
 
5
5
  module Miteru
6
6
  class Website
7
+ VALID_EXTENSIONS = Miteru.configuration.valid_extensions
8
+
7
9
  attr_reader :url
10
+
8
11
  def initialize(url)
9
12
  @url = url
10
13
  end
@@ -15,7 +18,7 @@ module Miteru
15
18
 
16
19
  def kits
17
20
  @kits ||= links.map do |link|
18
- kit = Kit.new(base_url: url, link: link.to_s)
21
+ kit = Kit.new(link)
19
22
  kit.valid? ? kit : nil
20
23
  end.compact
21
24
  end
@@ -33,7 +36,7 @@ module Miteru
33
36
  end
34
37
 
35
38
  def has_kits?
36
- ok? && index? && kits?
39
+ kits?
37
40
  rescue Addressable::URI::InvalidURIError, ArgumentError, Encoding::CompatibilityError, HTTP::Error, LL::ParserError, OpenSSL::SSL::SSLError => _e
38
41
  false
39
42
  end
@@ -46,6 +49,10 @@ module Miteru
46
49
  "It might contain #{noun}: #{filename_with_sizes}."
47
50
  end
48
51
 
52
+ def links
53
+ (href_links + possible_file_links).compact.uniq
54
+ end
55
+
49
56
  private
50
57
 
51
58
  def response
@@ -66,12 +73,31 @@ module Miteru
66
73
  nil
67
74
  end
68
75
 
69
- def links
70
- if doc
71
- doc.css("a").map { |a| a.get("href") }.compact
76
+ def href_links
77
+ if doc && ok? && index?
78
+ doc.css("a").map { |a| a.get("href") }.compact.map do |href|
79
+ href = href.start_with?("/") ? href : "/#{href}"
80
+ url + href
81
+ end
72
82
  else
73
83
  []
74
84
  end
85
+ rescue Addressable::URI::InvalidURIError, ArgumentError, Encoding::CompatibilityError, HTTP::Error, LL::ParserError, OpenSSL::SSL::SSLError => _e
86
+ []
87
+ end
88
+
89
+ def possible_file_links
90
+ uri = URI.parse(url)
91
+
92
+ segments = uri.path.split("/")
93
+ return [] if segments.length.zero?
94
+
95
+ last = segments.last
96
+ VALID_EXTENSIONS.map do |ext|
97
+ new_segments = segments[0..-2] + ["#{last}#{ext}"]
98
+ uri.path = new_segments.join("/")
99
+ uri.to_s
100
+ end
75
101
  end
76
102
  end
77
103
  end
@@ -29,7 +29,7 @@ Gem::Specification.new do |spec|
29
29
  spec.add_development_dependency "glint", "~> 0.1"
30
30
  spec.add_development_dependency "rake", "~> 13.0"
31
31
  spec.add_development_dependency "rspec", "~> 3.9"
32
- spec.add_development_dependency "vcr", "~> 5.1"
32
+ spec.add_development_dependency "vcr", "~> 6.0"
33
33
  spec.add_development_dependency "webmock", "~> 3.8"
34
34
 
35
35
  spec.add_dependency "colorize", "~> 0.8"
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: miteru
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.14.0
4
+ version: 0.14.5
5
5
  platform: ruby
6
6
  authors:
7
7
  - Manabu Niseki
8
- autorequire:
8
+ autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2020-05-14 00:00:00.000000000 Z
11
+ date: 2020-06-30 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -86,14 +86,14 @@ dependencies:
86
86
  requirements:
87
87
  - - "~>"
88
88
  - !ruby/object:Gem::Version
89
- version: '5.1'
89
+ version: '6.0'
90
90
  type: :development
91
91
  prerelease: false
92
92
  version_requirements: !ruby/object:Gem::Requirement
93
93
  requirements:
94
94
  - - "~>"
95
95
  - !ruby/object:Gem::Version
96
- version: '5.1'
96
+ version: '6.0'
97
97
  - !ruby/object:Gem::Dependency
98
98
  name: webmock
99
99
  requirement: !ruby/object:Gem::Requirement
@@ -264,7 +264,7 @@ homepage: https://github.com/ninoseki/miteru
264
264
  licenses:
265
265
  - MIT
266
266
  metadata: {}
267
- post_install_message:
267
+ post_install_message:
268
268
  rdoc_options: []
269
269
  require_paths:
270
270
  - lib
@@ -280,7 +280,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
280
280
  version: '0'
281
281
  requirements: []
282
282
  rubygems_version: 3.1.2
283
- signing_key:
283
+ signing_key:
284
284
  specification_version: 4
285
285
  summary: An experimental phishing kit detector
286
286
  test_files: []