miteru 0.9.1 → 0.9.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ca0bb26541d8e93bd9f566de4fa55440825d18363cb7f676062745ca378e4510
-  data.tar.gz: 23605936198cce076192bd4fca7e2eb704d9f89e08fa4870d2cee882f3272202
+  metadata.gz: a8370829b3ad8da53b7c73070fd972269b09ea27c6c7dce1a3da5d855f81da27
+  data.tar.gz: c017feecfeaa27912de5dc898f23b430652270a16852510785109d60ed04891a
 SHA512:
-  metadata.gz: 7ef37ae37316fbf4b53bb388e4dca0689789ee7521d1c56b875bf69a0aabf977e32bf928b8e323383f57ff2c391f8643cdabd6efc4827603e5d5b2b8275773f9
-  data.tar.gz: be702606497f66282717e9614b3b27ff7e2f18ac16046ef9a707b32e14837bac4ac7f4a4ad519fb704d5e8474c8b975765d52ad16c0a688729ed3bdddc0a28c6
+  metadata.gz: '0796be92ea1586ec57f12a4157097d65ea0d49e151eb8876eda3b406ca05908b0bc01ff021f270ab5a1467bda415e281909ef47502f1fb490d5285c69dd1bd9b'
+  data.tar.gz: edbc8c6cce3d85cbfb8bdac217d7ca0ee9d3dfc353a3b7a3c1c7f7ad132b7d9ee80960bda1bba638b859f53d2805a7634901d47fd9a913630b16b0552d463078

data/README.md

@@ -37,7 +37,7 @@ Usage:
   miteru execute
 
 Options:
-  [--auto-download], [--no-auto-download]              # Enable or disable auto-download of *.zip file(s)
+  [--auto-download], [--no-auto-download]              # Enable or disable auto-download of compressed file(s)
   [--directory-traveling], [--no-directory-traveling]  # Enable or disable directory traveling
   [--download-to=DOWNLOAD_TO]                          # Directory to download file(s)
                                                        # Default: /tmp

data/lib/miteru.rb

@@ -3,6 +3,7 @@
 require "miteru/error"
 require "miteru/http_client"
 require "miteru/website"
+require "miteru/downloader"
 require "miteru/crawler"
 require "miteru/cli"
 require "miteru/version"

data/lib/miteru/crawler.rb

@@ -10,7 +10,7 @@ module Miteru
   class Crawler
     attr_reader :auto_download
     attr_reader :directory_traveling
-    attr_reader :download_to
+    attr_reader :downloader
     attr_reader :size
     attr_reader :threads
     attr_reader :verbose
@@ -22,7 +22,7 @@ module Miteru
     def initialize(auto_download: false, directory_traveling: false, download_to: "/tmp", post_to_slack: false, size: 100, threads: 10, verbose: false)
       @auto_download = auto_download
       @directory_traveling = directory_traveling
-      @download_to = download_to
+      @downloader = Downloader.new(download_to)
       @post_to_slack = post_to_slack
       @size = size
       @threads = threads
@@ -59,6 +59,7 @@ module Miteru
       rescue URI::InvalidURIError => _
         return []
       end
+
       base = "#{uri.scheme}://#{uri.hostname}"
       return [base] unless directory_traveling
 
@@ -82,23 +83,17 @@ module Miteru
     def execute
       puts "Loaded #{suspicious_urls.length} URLs to crawl." if verbose
 
-      websites = []
       Parallel.each(suspicious_urls, in_threads: threads) do |url|
         website = Website.new(url)
-
         if website.has_kit?
           message = "#{website.url}: it might contain phishing kit(s) (#{website.compressed_files.join(', ')})."
           puts message.colorize(:light_red)
-          post_message_to_slack(website.message) if post_to_slack? && valid_slack_setting?
-          download_compressed_files(website.url, website.compressed_files, download_to) if auto_download?
+          post_a_message_to_slack(message) if post_to_slack? && valid_slack_setting?
+          downloader.download_compressed_files(website.url, website.compressed_files) if auto_download?
         else
           puts "#{website.url}: it doesn't contain a phishing kit." if verbose
         end
-        break
-      rescue StandardError => e
-        puts "Failed to load #{url} (#{e})" if verbose
       end
-      websites
     end
 
     def self.execute(auto_download: false, directory_traveling: false, download_to: "/tmp", post_to_slack: false, size: 100, threads: 10, verbose: false)
@@ -113,24 +108,7 @@ module Miteru
       ).execute
     end
 
-    def download_compressed_files(url, compressed_files, base_dir)
-      compressed_files.each do |path|
-        target_url = "#{url}/#{path}"
-        begin
-          download_file_path = HTTPClient.download(target_url, base_dir)
-          if duplicated?(download_file_path, base_dir)
-            puts "Do not download #{target_url} because there is a same hash file in the directory (SHA256: #{sha256(download_file_path)})."
-            FileUtils.rm download_file_path
-          else
-            puts "Download #{target_url} as #{download_file_path}"
-          end
-        rescue Down::Error => e
-          puts "Failed to download: #{target_url} (#{e})"
-        end
-      end
-    end
-
-    def post_to_slack(message)
+    def post_a_message_to_slack(message)
       webhook_url = ENV["SLACK_WEBHOOK_URL"]
       raise ArgumentError, "Please set the Slack webhook URL via SLACK_WEBHOOK_URL env" unless webhook_url
 
@@ -154,17 +132,6 @@ module Miteru
 
     private
 
-    def sha256(path)
-      digest = Digest::SHA256.file(path)
-      digest.hexdigest
-    end
-
-    def duplicated?(file_path, base_dir)
-      base = sha256(file_path)
-      sha256s = Dir.glob("#{base_dir}/*.zip").map { |path| sha256(path) }
-      sha256s.select { |sha256| sha256 == base }.length > 1
-    end
-
     def get(url)
       res = HTTP.follow(max_hops: 3).get(url)
       raise HTTPResponseError if res.code != 200

data/lib/miteru/downloader.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Miteru
+  class Downloader
+    attr_reader :base_dir
+
+    def initialize(base_dir = "/tmp")
+      @base_dir = base_dir
+      raise ArgumentError, "#{base_dir} is not existing." unless Dir.exist?(base_dir)
+    end
+
+    def download_compressed_files(url, compressed_files)
+      compressed_files.each do |path|
+        target_url = "#{url}/#{path}"
+        begin
+          download_file_path = HTTPClient.download(target_url, base_dir)
+          if duplicated?(download_file_path, base_dir)
+            puts "Do not download #{target_url} because there is a same hash file in the directory (SHA256: #{sha256(download_file_path)})."
+            FileUtils.rm download_file_path
+          else
+            puts "Download #{target_url} as #{download_file_path}"
+          end
+        rescue Down::Error => e
+          puts "Failed to download: #{target_url} (#{e})"
+        end
+      end
+    end
+
+    private
+
+    def sha256(path)
+      digest = Digest::SHA256.file(path)
+      digest.hexdigest
+    end
+
+    def duplicated?(file_path, base_dir)
+      base = sha256(file_path)
+      sha256s = Dir.glob("#{base_dir}/*.zip").map { |path| sha256(path) }
+      sha256s.select { |sha256| sha256 == base }.length > 1
+    end
+  end
+end

data/lib/miteru/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miteru
-  VERSION = "0.9.1"
+  VERSION = "0.9.2"
 end

data/lib/miteru/website.rb

@@ -7,7 +7,6 @@ module Miteru
     attr_reader :url
     def initialize(url)
       @url = url
-      build
     end
 
     def title
@@ -36,17 +35,9 @@ module Miteru
     end
 
     def has_kit?
-      @has_kit ||= ok? && index? && compressed_files?
-    end
-
-    def build
-      doc
-    end
-
-    def unbuild
-      @doc = nil
-      @response = nil
-      @compressed_files = nil
+      ok? && index? && compressed_files?
+    rescue StandardError => _
+      false
     end
 
     private

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: miteru
 version: !ruby/object:Gem::Version
-  version: 0.9.1
+  version: 0.9.2
 platform: ruby
 authors:
 - Manabu Niseki
@@ -213,6 +213,7 @@ files:
 - lib/miteru.rb
 - lib/miteru/cli.rb
 - lib/miteru/crawler.rb
+- lib/miteru/downloader.rb
 - lib/miteru/error.rb
 - lib/miteru/http_client.rb
 - lib/miteru/version.rb