miteru 0.14.1 → 0.14.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cbb47828d938e86289f845435bd9d342e18d7c50090e486b5681bf212dfcd84b
-  data.tar.gz: c416439bc1c80cfd4cf2de8bc9a7a6abb6ff46bb1af5d0a58ec0e71954584a4c
+  metadata.gz: acafc5c390603cb4e035ba592a47291eb5b93f20c1a6f4c12dbf22f40b15f3b4
+  data.tar.gz: bc8e05d8356ed633c45c1c241abb9972c79a85edab8f5d555cfce740d72f938f
 SHA512:
-  metadata.gz: f15f1770d870404e7b50cddb2e20d7e5d90adfab04283d5cb4364a2480f210b9c4869717d647bdc0174b5d0f0a9323903aac101f77d140ebad0c9523688b43d0
-  data.tar.gz: cb4a8e2c561aa53265362beec7e45445f6a6825bf991c7c4c559d40279dc84e138f54e300ab163f9db19dc891f2fe0a30c874d620d0464c9451c1e6c0e6e729e
+  metadata.gz: e191d8815c1eda041a9c64e2ef5c62a16da248ff5a19cbcd419ccdfa956963e0ed1177e83193c45df71971f8717ecd558a8a0b0d69a0d0ac8c7a2a4c9463ba87
+  data.tar.gz: b098a5efaa9eb18a618a5c3a42a0ce7b584ad37d39c7f154cc5afed9a11f5c0c2750215efa83b7d8a4e0109e32399dc37978da2a1bcf4aa2ff9c21f7454d974f

data/lib/miteru/feeds.rb

@@ -52,13 +52,15 @@ module Miteru
       urls = (0...segments.length).map do |idx|
         breakdowned_url = "#{base}#{segments[0..idx].join('/')}"
         breakdown = [breakdowned_url]
-        if idx > 0 && idx < segments.length - 1
+        if idx > 0 && idx < segments.length
+          next if segments[idx].nil? || invalid_extension?(segments[idx])
+
           VALID_EXTENSIONS.each do |ext|
             breakdown << "#{base}#{segments[0..idx - 1].join('/')}/#{segments[idx]}#{ext}"
           end
         end
         breakdown
-      end.flatten
+      end.flatten.compact
 
       urls.reject do |breakdowned_url|
         # Reject a url which ends with specific extension names

data/lib/miteru/http_client.rb

@@ -23,6 +23,15 @@ module Miteru
       destination
     end
 
+    def head(url, options = {})
+      options = options.merge default_options
+
+      HTTP.follow
+          .timeout(3)
+          .headers(urlscan_url?(url) ? urlscan_headers : default_headers)
+          .head(url, options)
+    end
+
     def get(url, options = {})
       options = options.merge default_options
 
@@ -48,6 +57,10 @@ module Miteru
       def post(url, options = {})
         new.post url, options
       end
+
+      def head(url, options = {})
+        new.head url, options
+      end
     end
 
     private

data/lib/miteru/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miteru
-  VERSION = "0.14.1"
+  VERSION = "0.14.2"
 end

data/lib/miteru/website.rb

@@ -4,6 +4,8 @@ require "oga"
 
 module Miteru
   class Website
+    VALID_EXTENSIONS = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
+
     attr_reader :url
     def initialize(url)
       @url = url
@@ -14,12 +16,25 @@ module Miteru
     end
 
     def kits
-      @kits ||= links.map do |link|
+      if ext?
+        return [] unless check(url)
+
+        link = url.split("/").last
+        base_url = url.split("/")[0..-2].join("/")
+        kit = Kit.new(base_url: base_url, link: link)
+        return kit.valid? ? [kit] : []
+      end
+
+      links.map do |link|
         kit = Kit.new(base_url: url, link: link.to_s)
         kit.valid? ? kit : nil
       end.compact
     end
 
+    def ext?
+      VALID_EXTENSIONS.any? { |ext| url.end_with?(ext) }
+    end
+
     def ok?
       response.code == 200
     end
@@ -33,6 +48,8 @@ module Miteru
     end
 
     def has_kits?
+      return kits? if ext?
+
       ok? && index? && kits?
     rescue Addressable::URI::InvalidURIError, ArgumentError, Encoding::CompatibilityError, HTTP::Error, LL::ParserError, OpenSSL::SSL::SSLError => _e
       false
@@ -52,6 +69,13 @@ module Miteru
       @response ||= get
     end
 
+    def check(url)
+      res = HTTPClient.head(url)
+      res.status.success?
+    rescue StandardError
+      false
+    end
+
     def get
       HTTPClient.get url
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: miteru
 version: !ruby/object:Gem::Version
-  version: 0.14.1
+  version: 0.14.2
 platform: ruby
 authors:
 - Manabu Niseki