miteru 0.14.1 → 0.14.2

This diff has not been reviewed by any users.
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: cbb47828d938e86289f845435bd9d342e18d7c50090e486b5681bf212dfcd84b
4
- data.tar.gz: c416439bc1c80cfd4cf2de8bc9a7a6abb6ff46bb1af5d0a58ec0e71954584a4c
3
+ metadata.gz: acafc5c390603cb4e035ba592a47291eb5b93f20c1a6f4c12dbf22f40b15f3b4
4
+ data.tar.gz: bc8e05d8356ed633c45c1c241abb9972c79a85edab8f5d555cfce740d72f938f
5
5
  SHA512:
6
- metadata.gz: f15f1770d870404e7b50cddb2e20d7e5d90adfab04283d5cb4364a2480f210b9c4869717d647bdc0174b5d0f0a9323903aac101f77d140ebad0c9523688b43d0
7
- data.tar.gz: cb4a8e2c561aa53265362beec7e45445f6a6825bf991c7c4c559d40279dc84e138f54e300ab163f9db19dc891f2fe0a30c874d620d0464c9451c1e6c0e6e729e
6
+ metadata.gz: e191d8815c1eda041a9c64e2ef5c62a16da248ff5a19cbcd419ccdfa956963e0ed1177e83193c45df71971f8717ecd558a8a0b0d69a0d0ac8c7a2a4c9463ba87
7
+ data.tar.gz: b098a5efaa9eb18a618a5c3a42a0ce7b584ad37d39c7f154cc5afed9a11f5c0c2750215efa83b7d8a4e0109e32399dc37978da2a1bcf4aa2ff9c21f7454d974f
@@ -52,13 +52,15 @@ module Miteru
52
52
  urls = (0...segments.length).map do |idx|
53
53
  breakdowned_url = "#{base}#{segments[0..idx].join('/')}"
54
54
  breakdown = [breakdowned_url]
55
- if idx > 0 && idx < segments.length - 1
55
+ if idx > 0 && idx < segments.length
56
+ next if segments[idx].nil? || invalid_extension?(segments[idx])
57
+
56
58
  VALID_EXTENSIONS.each do |ext|
57
59
  breakdown << "#{base}#{segments[0..idx - 1].join('/')}/#{segments[idx]}#{ext}"
58
60
  end
59
61
  end
60
62
  breakdown
61
- end.flatten
63
+ end.flatten.compact
62
64
 
63
65
  urls.reject do |breakdowned_url|
64
66
  # Reject a url which ends with specific extension names
@@ -23,6 +23,15 @@ module Miteru
23
23
  destination
24
24
  end
25
25
 
26
+ def head(url, options = {})
27
+ options = options.merge default_options
28
+
29
+ HTTP.follow
30
+ .timeout(3)
31
+ .headers(urlscan_url?(url) ? urlscan_headers : default_headers)
32
+ .head(url, options)
33
+ end
34
+
26
35
  def get(url, options = {})
27
36
  options = options.merge default_options
28
37
 
@@ -48,6 +57,10 @@ module Miteru
48
57
  def post(url, options = {})
49
58
  new.post url, options
50
59
  end
60
+
61
+ def head(url, options = {})
62
+ new.head url, options
63
+ end
51
64
  end
52
65
 
53
66
  private
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Miteru
4
- VERSION = "0.14.1"
4
+ VERSION = "0.14.2"
5
5
  end
@@ -4,6 +4,8 @@ require "oga"
4
4
 
5
5
  module Miteru
6
6
  class Website
7
+ VALID_EXTENSIONS = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
8
+
7
9
  attr_reader :url
8
10
  def initialize(url)
9
11
  @url = url
@@ -14,12 +16,25 @@ module Miteru
14
16
  end
15
17
 
16
18
  def kits
17
- @kits ||= links.map do |link|
19
+ if ext?
20
+ return [] unless check(url)
21
+
22
+ link = url.split("/").last
23
+ base_url = url.split("/")[0..-2].join("/")
24
+ kit = Kit.new(base_url: base_url, link: link)
25
+ return kit.valid? ? [kit] : []
26
+ end
27
+
28
+ links.map do |link|
18
29
  kit = Kit.new(base_url: url, link: link.to_s)
19
30
  kit.valid? ? kit : nil
20
31
  end.compact
21
32
  end
22
33
 
34
+ def ext?
35
+ VALID_EXTENSIONS.any? { |ext| url.end_with?(ext) }
36
+ end
37
+
23
38
  def ok?
24
39
  response.code == 200
25
40
  end
@@ -33,6 +48,8 @@ module Miteru
33
48
  end
34
49
 
35
50
  def has_kits?
51
+ return kits? if ext?
52
+
36
53
  ok? && index? && kits?
37
54
  rescue Addressable::URI::InvalidURIError, ArgumentError, Encoding::CompatibilityError, HTTP::Error, LL::ParserError, OpenSSL::SSL::SSLError => _e
38
55
  false
@@ -52,6 +69,13 @@ module Miteru
52
69
  @response ||= get
53
70
  end
54
71
 
72
+ def check(url)
73
+ res = HTTPClient.head(url)
74
+ res.status.success?
75
+ rescue StandardError
76
+ false
77
+ end
78
+
55
79
  def get
56
80
  HTTPClient.get url
57
81
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: miteru
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.14.1
4
+ version: 0.14.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Manabu Niseki