miteru 0.9.0 → 0.9.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 48d47d401250750c0ffa68ccf5f58602a1e0fa4918b0b812aebcd0c60cdfe484
-  data.tar.gz: 9acbfd9d4465c25ddb20186fe57a2d0a84ba74ba24fa2e55b2b2103b5a4d12f9
+  metadata.gz: ca0bb26541d8e93bd9f566de4fa55440825d18363cb7f676062745ca378e4510
+  data.tar.gz: 23605936198cce076192bd4fca7e2eb704d9f89e08fa4870d2cee882f3272202
 SHA512:
-  metadata.gz: 7690a2497203522b7c9469656a5e1ea9fd2e72671555548a1d0833841d8b3bce34a60c22b2e9597b94add35342dbae0b2cb3fd58fdb3ec08b9e16375f1ca6044
-  data.tar.gz: 27d5aef80aa2bae48091bfaf398fae956e7226384414bac4505af3752090a722c1be1b421d3260773dd40d7d7c2dc65281b5555bbade267f83554d267371cd85
+  metadata.gz: 7ef37ae37316fbf4b53bb388e4dca0689789ee7521d1c56b875bf69a0aabf977e32bf928b8e323383f57ff2c391f8643cdabd6efc4827603e5d5b2b8275773f9
+  data.tar.gz: be702606497f66282717e9614b3b27ff7e2f18ac16046ef9a707b32e14837bac4ac7f4a4ad519fb704d5e8474c8b975765d52ad16c0a688729ed3bdddc0a28c6

data/lib/miteru/cli.rb

@@ -17,64 +17,7 @@ module Miteru
     method_option :verbose, type: :boolean, default: true
     desc "execute", "Execute the crawler"
     def execute
-      websites = Crawler.execute(
-        directory_traveling: options[:directory_traveling],
-        size: options[:size],
-        threads: options[:threads],
-        verbose: options[:verbose]
-      )
-      websites.each do |website|
-        next unless website.has_kit?
-
-        message = "#{website.url}: it might contain phishing kit(s) (#{website.compressed_files.join(', ')})."
-        puts message.colorize(:light_red)
-        post_to_slack(website.message) if options[:post_to_slack] && valid_slack_setting?
-        download_compressed_files(website.url, website.compressed_files, options[:download_to]) if options[:auto_download]
-      end
-    end
-
-    no_commands do
-      def download_compressed_files(url, compressed_files, base_dir)
-        compressed_files.each do |path|
-          target_url = "#{url}/#{path}"
-          begin
-            download_file_path = HTTPClient.download(target_url, base_dir)
-            if duplicated?(download_file_path, base_dir)
-              puts "Do not download #{target_url} because there is a same hash file in the directory (SHA256: #{sha256(download_file_path)})."
-              FileUtils.rm download_file_path
-            else
-              puts "Download #{target_url} as #{download_file_path}"
-            end
-          rescue Down::Error => e
-            puts "Failed to download: #{target_url} (#{e})"
-          end
-        end
-      end
-
-      def sha256(path)
-        digest = Digest::SHA256.file(path)
-        digest.hexdigest
-      end
-
-      def duplicated?(file_path, base_dir)
-        base = sha256(file_path)
-        sha256s = Dir.glob("#{base_dir}/*.zip").map { |path| sha256(path) }
-        sha256s.select { |sha256| sha256 == base }.length > 1
-      end
-
-      def valid_slack_setting?
-        ENV["SLACK_WEBHOOK_URL"] != nil
-      end
-
-      def post_to_slack(message)
-        webhook_url = ENV["SLACK_WEBHOOK_URL"]
-        raise ArgumentError, "Please set the Slack webhook URL via SLACK_WEBHOOK_URL env" unless webhook_url
-
-        channel = ENV["SLACK_CHANNEL"] || "#general"
-
-        payload = { text: message, channel: channel }
-        HTTP.post(webhook_url, json: payload)
-      end
+      Crawler.execute options.map { |k, v| [k.to_sym, v] }.to_h
     end
   end
 end

data/lib/miteru/crawler.rb

@@ -3,12 +3,14 @@
 require "csv"
 require "http"
 require "json"
-require "thread/pool"
+require "parallel"
 require "uri"
 
 module Miteru
   class Crawler
+    attr_reader :auto_download
     attr_reader :directory_traveling
+    attr_reader :download_to
     attr_reader :size
     attr_reader :threads
     attr_reader :verbose
@@ -17,8 +19,11 @@ module Miteru
     OPENPHISH_ENDPOINT = "https://openphish.com"
     PHISHTANK_ENDPOINT = "http://data.phishtank.com"
 
-    def initialize(directory_traveling: false, size: 100, threads: 10, verbose: false)
+    def initialize(auto_download: false, directory_traveling: false, download_to: "/tmp", post_to_slack: false, size: 100, threads: 10, verbose: false)
+      @auto_download = auto_download
       @directory_traveling = directory_traveling
+      @download_to = download_to
+      @post_to_slack = post_to_slack
       @size = size
       @threads = threads
       @verbose = verbose
@@ -69,7 +74,7 @@ module Miteru
 
     def suspicious_urls
       @suspicious_urls ||= [].tap do |arr|
-        urls = (urlscan_feed + openphish_feed + phishtank_feed)
+        urls = (urlscan_feed + openphish_feed + phishtank_feed).select { |url| url.start_with?("http://", "https://") }
         urls.map { |url| breakdown(url) }.flatten.uniq.sort.each { |url| arr << url }
       end
     end
@@ -77,31 +82,89 @@ module Miteru
     def execute
       puts "Loaded #{suspicious_urls.length} URLs to crawl." if verbose
 
-      pool = Thread.pool(threads)
       websites = []
+      Parallel.each(suspicious_urls, in_threads: threads) do |url|
+        website = Website.new(url)
+
+        if website.has_kit?
+          message = "#{website.url}: it might contain phishing kit(s) (#{website.compressed_files.join(', ')})."
+          puts message.colorize(:light_red)
+          post_message_to_slack(website.message) if post_to_slack? && valid_slack_setting?
+          download_compressed_files(website.url, website.compressed_files, download_to) if auto_download?
+        else
+          puts "#{website.url}: it doesn't contain a phishing kit." if verbose
+        end
+        break
+      rescue StandardError => e
+        puts "Failed to load #{url} (#{e})" if verbose
+      end
+      websites
+    end
 
-      suspicious_urls.each do |url|
-        pool.process do
-          website = Website.new(url)
-          if website.has_kit?
-            websites << website
+    def self.execute(auto_download: false, directory_traveling: false, download_to: "/tmp", post_to_slack: false, size: 100, threads: 10, verbose: false)
+      new(
+        auto_download: auto_download,
+        directory_traveling: directory_traveling,
+        download_to: download_to,
+        post_to_slack: post_to_slack,
+        size: size,
+        threads: threads,
+        verbose: verbose
+      ).execute
+    end
+
+    def download_compressed_files(url, compressed_files, base_dir)
+      compressed_files.each do |path|
+        target_url = "#{url}/#{path}"
+        begin
+          download_file_path = HTTPClient.download(target_url, base_dir)
+          if duplicated?(download_file_path, base_dir)
+            puts "Do not download #{target_url} because there is a same hash file in the directory (SHA256: #{sha256(download_file_path)})."
+            FileUtils.rm download_file_path
           else
-            puts "#{website.url}: it doesn't contain a phishing kit." if verbose
-            website.unbuild
+            puts "Download #{target_url} as #{download_file_path}"
           end
+        rescue Down::Error => e
+          puts "Failed to download: #{target_url} (#{e})"
         end
       end
-      pool.shutdown
+    end
 
-      websites
+    def post_to_slack(message)
+      webhook_url = ENV["SLACK_WEBHOOK_URL"]
+      raise ArgumentError, "Please set the Slack webhook URL via SLACK_WEBHOOK_URL env" unless webhook_url
+
+      channel = ENV["SLACK_CHANNEL"] || "#general"
+
+      payload = { text: message, channel: channel }
+      HTTP.post(webhook_url, json: payload)
+    end
+
+    def post_to_slack?
+      @post_to_slack
+    end
+
+    def auto_download?
+      @auto_download
     end
 
-    def self.execute(directory_traveling: false, size: 100, threads: 10, verbose: false)
-      new(directory_traveling: directory_traveling, size: size, threads: threads, verbose: verbose).execute
+    def valid_slack_setting?
+      ENV["SLACK_WEBHOOK_URL"] != nil
     end
 
     private
 
+    def sha256(path)
+      digest = Digest::SHA256.file(path)
+      digest.hexdigest
+    end
+
+    def duplicated?(file_path, base_dir)
+      base = sha256(file_path)
+      sha256s = Dir.glob("#{base_dir}/*.zip").map { |path| sha256(path) }
+      sha256s.select { |sha256| sha256 == base }.length > 1
+    end
+
     def get(url)
       res = HTTP.follow(max_hops: 3).get(url)
       raise HTTPResponseError if res.code != 200

data/lib/miteru/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miteru
-  VERSION = "0.9.0"
+  VERSION = "0.9.1"
 end

data/miteru.gemspec

@@ -36,6 +36,6 @@ Gem::Specification.new do |spec|
   spec.add_dependency "down", "~> 4.5"
   spec.add_dependency "http", "~> 3.3"
   spec.add_dependency "oga", "~> 2.15"
+  spec.add_dependency "parallel", "~> 1.12"
   spec.add_dependency "thor", "~> 0.19"
-  spec.add_dependency "thread", "~> 0.2.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miteru
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 0.9.1
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-10-16 00:00:00.000000000 Z
+date: 2018-10-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -165,33 +165,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.15'
 - !ruby/object:Gem::Dependency
-  name: thor
+  name: parallel
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.19'
+        version: '1.12'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.19'
+        version: '1.12'
 - !ruby/object:Gem::Dependency
-  name: thread
+  name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.2
+        version: '0.19'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.2
+        version: '0.19'
 description: An experimental phishing kit detector
 email:
 - manabu.niseki@gmail.com