miteru 0.14.2 → 0.14.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: acafc5c390603cb4e035ba592a47291eb5b93f20c1a6f4c12dbf22f40b15f3b4
-  data.tar.gz: bc8e05d8356ed633c45c1c241abb9972c79a85edab8f5d555cfce740d72f938f
+  metadata.gz: c1ae4b442c2963ff387cc1bf3bf6390a393c2f8ba416276f2dc0cc48f2ceb86b
+  data.tar.gz: ce3baa2e515837cf722a4bd90650f558a337cde2e2dd2ecef976620f7d20eddb
 SHA512:
-  metadata.gz: e191d8815c1eda041a9c64e2ef5c62a16da248ff5a19cbcd419ccdfa956963e0ed1177e83193c45df71971f8717ecd558a8a0b0d69a0d0ac8c7a2a4c9463ba87
-  data.tar.gz: b098a5efaa9eb18a618a5c3a42a0ce7b584ad37d39c7f154cc5afed9a11f5c0c2750215efa83b7d8a4e0109e32399dc37978da2a1bcf4aa2ff9c21f7454d974f
+  metadata.gz: 40afc8ff440ffad5be4e4b7efdb7670c6df1a9e05d503930424e53a6c57fcfd3270239f39a92de2ccac149e8bc6175c87833183ad822d900d526490f067c06dc
+  data.tar.gz: 2ad37a6b2ebfaf78451ba35fa70d826ad95202cf077fdba5ed02cbebd8c74104c6f7f2e49da1ea26a9d42890bbd0b4d7b76e06787cab3697ba43f441a5825230

data/docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM ruby:2.6-alpine3.10
+FROM ruby:2.7-alpine3.10
 RUN apk --no-cache add git build-base ruby-dev \
   && cd /tmp/ \
   && git clone https://github.com/ninoseki/miteru.git \

data/exe/miteru

@@ -5,4 +5,5 @@ $LOAD_PATH.unshift("#{__dir__}/../lib")
 
 require "miteru"
 
-Miteru::CLI.start
+ARGV.unshift(Miteru::CLI.default_task) unless Miteru::CLI.all_tasks.key?(ARGV[0])
+Miteru::CLI.start(ARGV)

data/lib/miteru/cli.rb

@@ -29,5 +29,13 @@ module Miteru
 
       Crawler.execute
     end
+
+    default_command :execute
+
+    class << self
+      def exit_on_failure?
+        true
+      end
+    end
   end
 end

data/lib/miteru/configuration.rb

@@ -28,6 +28,10 @@ module Miteru
     # @return [Boolean]
     attr_accessor :verbose
 
+    attr_reader :valid_extensions
+
+    attr_reader :valid_mime_types
+
     def initialize
       @auto_download = false
       @ayashige = false
@@ -37,6 +41,9 @@ module Miteru
       @size = 100
       @threads = Parallel.processor_count
       @verbose = false
+
+      @valid_extensions = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
+      @valid_mime_types = ["application/zip", "application/vnd.rar", "application/x-7z-compressed", "application/x-tar", "application/gzip"]
     end
 
     def auto_download?

data/lib/miteru/feeds.rb

@@ -10,7 +10,6 @@ require_relative "./feeds/urlscan_pro"
 module Miteru
   class Feeds
     IGNORE_EXTENSIONS = %w(.htm .html .php .asp .aspx .exe .txt).freeze
-    VALID_EXTENSIONS = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
 
     def initialize
       @feeds = [
@@ -49,18 +48,7 @@ module Miteru
       segments = uri.path.split("/")
       return [base] if segments.length.zero?
 
-      urls = (0...segments.length).map do |idx|
-        breakdowned_url = "#{base}#{segments[0..idx].join('/')}"
-        breakdown = [breakdowned_url]
-        if idx > 0 && idx < segments.length
-          next if segments[idx].nil? || invalid_extension?(segments[idx])
-
-          VALID_EXTENSIONS.each do |ext|
-            breakdown << "#{base}#{segments[0..idx - 1].join('/')}/#{segments[idx]}#{ext}"
-          end
-        end
-        breakdown
-      end.flatten.compact
+      urls = (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join('/')}" }
 
       urls.reject do |breakdowned_url|
         # Reject a url which ends with specific extension names

data/lib/miteru/feeds/phishing_database.rb

@@ -6,7 +6,7 @@ require "uri"
 module Miteru
   class Feeds
     class PhishingDatabase < Feed
-      URL = "https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/phishing-links-NEW-today.txt"
+      URL = "https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/phishing-links-ACTIVE-NOW.txt"
 
       def urls
         body = get(URL)

data/lib/miteru/http_client.rb

@@ -18,7 +18,7 @@ module Miteru
     end
 
     def download(url, destination)
-      down = Down::Http.new(default_options) { |client| client.headers(default_headers) }
+      down = Down::Http.new(**default_options) { |client| client.headers(**default_headers) }
       down.download(url, destination: destination)
       destination
     end

data/lib/miteru/kit.rb

@@ -5,37 +5,44 @@ require "securerandom"
 
 module Miteru
   class Kit
-    VALID_EXTENSIONS = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
+    VALID_EXTENSIONS = Miteru.configuration.valid_extensions
+    VALID_MIME_TYPES = Miteru.configuration.valid_mime_types
 
-    attr_reader :base_url, :link
+    attr_reader :url
 
-    def initialize(base_url:, link:)
-      @base_url = base_url
-      @link = link.start_with?("/") ? link[1..-1] : link
+    attr_reader :status
+    attr_reader :content_length
+    attr_reader :mime_type
+
+    def initialize(url)
+      @url = url
+
+      @content_length = nil
+      @mime_type = nil
+      @status = nil
     end
 
-    def valid?
-      VALID_EXTENSIONS.include? extname
+    def valid?;
+      # make a HEAD request for the validation
+      before_validation
+
+      valid_ext? && reachable? && valid_mime_type? && valid_content_length?
     end
 
     def extname
-      return ".tar.gz" if link.end_with?("tar.gz")
+      return ".tar.gz" if url.end_with?("tar.gz")
 
-      File.extname(link)
+      File.extname(url)
     end
 
     def basename
-      File.basename(link)
+      File.basename(url)
     end
 
     def filename
       CGI.unescape basename
     end
 
-    def url
-      "#{base_url}/#{basename}"
-    end
-
     def download_filepath
       "#{base_dir}/#{download_filename}"
     end
@@ -59,7 +66,7 @@ module Miteru
     end
 
     def hostname
-      URI(base_url).hostname
+      URI(url).hostname
     end
 
     def download_filename
@@ -69,5 +76,30 @@ module Miteru
     def base_dir
       @base_dir ||= Miteru.configuration.download_to
     end
+
+    def valid_ext?
+      VALID_EXTENSIONS.include? extname
+    end
+
+    def before_validation
+      res = HTTPClient.head(url)
+      @content_length = res.content_length
+      @mime_type = res.content_type.mime_type.to_s
+      @status = res.status
+    rescue StandardError
+      # do nothing
+    end
+
+    def reachable?
+      status&.success?
+    end
+
+    def valid_mime_type?
+      VALID_MIME_TYPES.include? mime_type
+    end
+
+    def valid_content_length?
+      content_length.to_i > 0
+    end
   end
 end

data/lib/miteru/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miteru
-  VERSION = "0.14.2"
+  VERSION = "0.14.7"
 end

data/lib/miteru/website.rb

@@ -4,9 +4,10 @@ require "oga"
 
 module Miteru
   class Website
-    VALID_EXTENSIONS = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
+    VALID_EXTENSIONS = Miteru.configuration.valid_extensions
 
     attr_reader :url
+
     def initialize(url)
       @url = url
     end
@@ -16,25 +17,12 @@ module Miteru
     end
 
     def kits
-      if ext?
-        return [] unless check(url)
-
-        link = url.split("/").last
-        base_url = url.split("/")[0..-2].join("/")
-        kit = Kit.new(base_url: base_url, link: link)
-        return kit.valid? ? [kit] : []
-      end
-
-      links.map do |link|
-        kit = Kit.new(base_url: url, link: link.to_s)
+      @kits ||= links.map do |link|
+        kit = Kit.new(link)
         kit.valid? ? kit : nil
       end.compact
     end
 
-    def ext?
-      VALID_EXTENSIONS.any? { |ext| url.end_with?(ext) }
-    end
-
     def ok?
       response.code == 200
     end
@@ -48,9 +36,7 @@ module Miteru
     end
 
     def has_kits?
-      return kits? if ext?
-
-      ok? && index? && kits?
+      kits?
     rescue Addressable::URI::InvalidURIError, ArgumentError, Encoding::CompatibilityError, HTTP::Error, LL::ParserError, OpenSSL::SSL::SSLError => _e
       false
     end
@@ -63,19 +49,16 @@ module Miteru
       "It might contain #{noun}: #{filename_with_sizes}."
     end
 
+    def links
+      (href_links + possible_file_links).compact.uniq
+    end
+
     private
 
     def response
       @response ||= get
     end
 
-    def check(url)
-      res = HTTPClient.head(url)
-      res.status.success?
-    rescue StandardError
-      false
-    end
-
     def get
       HTTPClient.get url
     end
@@ -90,12 +73,31 @@ module Miteru
       nil
     end
 
-    def links
-      if doc
-        doc.css("a").map { |a| a.get("href") }.compact
+    def href_links
+      if doc && ok? && index?
+        doc.css("a").map { |a| a.get("href") }.compact.map do |href|
+          href = href.start_with?("/") ? href : "/#{href}"
+          url + href
+        end
       else
         []
       end
+    rescue Addressable::URI::InvalidURIError, ArgumentError, Encoding::CompatibilityError, HTTP::Error, LL::ParserError, OpenSSL::SSL::SSLError => _e
+      []
+    end
+
+    def possible_file_links
+      uri = URI.parse(url)
+
+      segments = uri.path.split("/")
+      return [] if segments.length.zero?
+
+      last = segments.last
+      VALID_EXTENSIONS.map do |ext|
+        new_segments = segments[0..-2] + ["#{last}#{ext}"]
+        uri.path = new_segments.join("/")
+        uri.to_s
+      end
     end
   end
 end

data/miteru.gemspec

@@ -30,14 +30,14 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.9"
   spec.add_development_dependency "vcr", "~> 6.0"
-  spec.add_development_dependency "webmock", "~> 3.8"
+  spec.add_development_dependency "webmock", "~> 3.9"
 
   spec.add_dependency "colorize", "~> 0.8"
-  spec.add_dependency "down", "~> 5.1"
+  spec.add_dependency "down", "~> 5.2"
   spec.add_dependency "http", "~> 4.4"
-  spec.add_dependency "oga", "~> 3.2"
+  spec.add_dependency "oga", "~> 3.3"
   spec.add_dependency "parallel", "~> 1.19"
   spec.add_dependency "slack-notifier", "~> 2.3"
   spec.add_dependency "thor", "~> 1.0"
-  spec.add_dependency "urlscan", "~> 0.5"
+  spec.add_dependency "urlscan", "~> 0.6"
 end

data/renovate.json

@@ -0,0 +1,5 @@
+{
+  "extends": [
+    "config:base"
+  ]
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miteru
 version: !ruby/object:Gem::Version
-  version: 0.14.2
+  version: 0.14.7
 platform: ruby
 authors:
 - Manabu Niseki
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-06-23 00:00:00.000000000 Z
+date: 2020-10-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -100,14 +100,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.8'
+        version: '3.9'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.8'
+        version: '3.9'
 - !ruby/object:Gem::Dependency
   name: colorize
   requirement: !ruby/object:Gem::Requirement
@@ -128,14 +128,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.1'
+        version: '5.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.1'
+        version: '5.2'
 - !ruby/object:Gem::Dependency
   name: http
   requirement: !ruby/object:Gem::Requirement
@@ -156,14 +156,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.2'
+        version: '3.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.2'
+        version: '3.3'
 - !ruby/object:Gem::Dependency
   name: parallel
   requirement: !ruby/object:Gem::Requirement
@@ -212,14 +212,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: '0.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: '0.6'
 description: An experimental phishing kit detector
 email:
 - manabu.niseki@gmail.com
@@ -259,12 +259,13 @@ files:
 - lib/miteru/version.rb
 - lib/miteru/website.rb
 - miteru.gemspec
+- renovate.json
 - screenshots/slack.png
 homepage: https://github.com/ninoseki/miteru
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -280,7 +281,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.1.2
-signing_key: 
+signing_key:
 specification_version: 4
 summary: An experimental phishing kit detector
 test_files: []