miteru 0.14.0 → 0.14.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5bcc07aabe006f3107c410263f09c4580081933b7a7aebfa7344977f6fd05dc6
-  data.tar.gz: 49cbc8f6efb26eb47ea776c46986bf949f77371a31109e6298bfd11fe7ca670f
+  metadata.gz: 1cf6445b9388cdcfb797cdb8c91615a288757508874d0550bab8c6b052c375d6
+  data.tar.gz: dd9f428317a0f325c2c467234dda21f05bb7cd616114f49adeec9374a4680451
 SHA512:
-  metadata.gz: 54b51311d0f982c2da7221c09384847d6bd526356158dc56948f1d898f3631ef22f3005a041ca0ebc31cfc22bcafc0d64047d95392857f79c8611dd2802bb366
-  data.tar.gz: c61864c67eb49c79aaa180827610fab24e2fa3de7e14e8396c9ff6ca60439cbe271e313aff6618a8986c2a8eb8d889e018ef062aa92dd2c62afe7ea54bfd7f98
+  metadata.gz: 5f740f27cacdb1020a49bc787aa8486cbe6b6ac6fe5e4ad3093af0601928dcd12c78895c39c548a28de18fc35766dbb89507687c81e2bb162d6e69e111242fbd
+  data.tar.gz: a110c028b6129a2d0bc903962d1f945be96426b8c909ae6e34bb1955d3ca657b1c9ef5a48bc1e147c54f992517cc86c0ca18047a79cdcb2b2a95bda5c5148caa

data/lib/miteru/configuration.rb

@@ -28,6 +28,10 @@ module Miteru
     # @return [Boolean]
     attr_accessor :verbose
 
+    attr_reader :valid_extensions
+
+    attr_reader :valid_mime_types
+
     def initialize
       @auto_download = false
       @ayashige = false
@@ -37,6 +41,9 @@ module Miteru
       @size = 100
       @threads = Parallel.processor_count
       @verbose = false
+
+      @valid_extensions = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
+      @valid_mime_types = ["application/zip", "application/vnd.rar", "application/x-7z-compressed", "application/x-tar", "application/gzip"]
     end
 
     def auto_download?

data/lib/miteru/feeds.rb

@@ -49,6 +49,7 @@ module Miteru
       return [base] if segments.length.zero?
 
       urls = (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join('/')}" }
+
       urls.reject do |breakdowned_url|
         # Reject a url which ends with specific extension names
         invalid_extension? breakdowned_url

data/lib/miteru/http_client.rb

@@ -18,11 +18,20 @@ module Miteru
     end
 
     def download(url, destination)
-      down = Down::Http.new(default_options) { |client| client.headers(default_headers) }
+      down = Down::Http.new(**default_options) { |client| client.headers(**default_headers) }
       down.download(url, destination: destination)
       destination
     end
 
+    def head(url, options = {})
+      options = options.merge default_options
+
+      HTTP.follow
+          .timeout(3)
+          .headers(urlscan_url?(url) ? urlscan_headers : default_headers)
+          .head(url, options)
+    end
+
     def get(url, options = {})
       options = options.merge default_options
 
@@ -48,6 +57,10 @@ module Miteru
       def post(url, options = {})
         new.post url, options
       end
+
+      def head(url, options = {})
+        new.head url, options
+      end
     end
 
     private

data/lib/miteru/kit.rb

@@ -5,37 +5,33 @@ require "securerandom"
 
 module Miteru
   class Kit
-    VALID_EXTENSIONS = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
+    VALID_EXTENSIONS = Miteru.configuration.valid_extensions
+    VALID_MIME_TYPES = Miteru.configuration.valid_mime_types
 
-    attr_reader :base_url, :link
+    attr_reader :url
 
-    def initialize(base_url:, link:)
-      @base_url = base_url
-      @link = link.start_with?("/") ? link[1..-1] : link
+    def initialize(url)
+      @url = url
     end
 
-    def valid?
-      VALID_EXTENSIONS.include? extname
+    def valid?;
+      valid_ext? && reachable_and_valid_mime_type?
     end
 
     def extname
-      return ".tar.gz" if link.end_with?("tar.gz")
+      return ".tar.gz" if url.end_with?("tar.gz")
 
-      File.extname(link)
+      File.extname(url)
     end
 
     def basename
-      File.basename(link)
+      File.basename(url)
     end
 
     def filename
       CGI.unescape basename
     end
 
-    def url
-      "#{base_url}/#{basename}"
-    end
-
     def download_filepath
       "#{base_dir}/#{download_filename}"
     end
@@ -59,7 +55,7 @@ module Miteru
     end
 
     def hostname
-      URI(base_url).hostname
+      URI(url).hostname
     end
 
     def download_filename
@@ -69,5 +65,25 @@ module Miteru
     def base_dir
       @base_dir ||= Miteru.configuration.download_to
     end
+
+    def valid_ext?
+      VALID_EXTENSIONS.include? extname
+    end
+
+    def reachable?(response)
+      response.status.success?
+    end
+
+    def valid_mime_type?(response)
+      mime_type = response.content_type.mime_type.to_s
+      VALID_MIME_TYPES.include? mime_type
+    end
+
+    def reachable_and_valid_mime_type?
+      res = HTTPClient.head(url)
+      reachable?(res) && valid_mime_type?(res)
+    rescue StandardError
+      false
+    end
   end
 end

data/lib/miteru/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miteru
-  VERSION = "0.14.0"
+  VERSION = "0.14.5"
 end

data/lib/miteru/website.rb

@@ -4,7 +4,10 @@ require "oga"
 
 module Miteru
   class Website
+    VALID_EXTENSIONS = Miteru.configuration.valid_extensions
+
     attr_reader :url
+
     def initialize(url)
       @url = url
     end
@@ -15,7 +18,7 @@ module Miteru
 
     def kits
       @kits ||= links.map do |link|
-        kit = Kit.new(base_url: url, link: link.to_s)
+        kit = Kit.new(link)
         kit.valid? ? kit : nil
       end.compact
     end
@@ -33,7 +36,7 @@ module Miteru
     end
 
     def has_kits?
-      ok? && index? && kits?
+      kits?
     rescue Addressable::URI::InvalidURIError, ArgumentError, Encoding::CompatibilityError, HTTP::Error, LL::ParserError, OpenSSL::SSL::SSLError => _e
       false
     end
@@ -46,6 +49,10 @@ module Miteru
       "It might contain #{noun}: #{filename_with_sizes}."
     end
 
+    def links
+      (href_links + possible_file_links).compact.uniq
+    end
+
     private
 
     def response
@@ -66,12 +73,31 @@ module Miteru
       nil
     end
 
-    def links
-      if doc
-        doc.css("a").map { |a| a.get("href") }.compact
+    def href_links
+      if doc && ok? && index?
+        doc.css("a").map { |a| a.get("href") }.compact.map do |href|
+          href = href.start_with?("/") ? href : "/#{href}"
+          url + href
+        end
       else
         []
       end
+    rescue Addressable::URI::InvalidURIError, ArgumentError, Encoding::CompatibilityError, HTTP::Error, LL::ParserError, OpenSSL::SSL::SSLError => _e
+      []
+    end
+
+    def possible_file_links
+      uri = URI.parse(url)
+
+      segments = uri.path.split("/")
+      return [] if segments.length.zero?
+
+      last = segments.last
+      VALID_EXTENSIONS.map do |ext|
+        new_segments = segments[0..-2] + ["#{last}#{ext}"]
+        uri.path = new_segments.join("/")
+        uri.to_s
+      end
     end
   end
 end

data/miteru.gemspec

@@ -29,7 +29,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "glint", "~> 0.1"
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.9"
-  spec.add_development_dependency "vcr", "~> 5.1"
+  spec.add_development_dependency "vcr", "~> 6.0"
   spec.add_development_dependency "webmock", "~> 3.8"
 
   spec.add_dependency "colorize", "~> 0.8"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miteru
 version: !ruby/object:Gem::Version
-  version: 0.14.0
+  version: 0.14.5
 platform: ruby
 authors:
 - Manabu Niseki
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-05-14 00:00:00.000000000 Z
+date: 2020-06-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -86,14 +86,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.1'
+        version: '6.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.1'
+        version: '6.0'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
@@ -264,7 +264,7 @@ homepage: https://github.com/ninoseki/miteru
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -280,7 +280,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.1.2
-signing_key: 
+signing_key:
 specification_version: 4
 summary: An experimental phishing kit detector
 test_files: []