miteru 0.8.0 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: efab1503dd6c96a4376d6baed09eadccde40d9a43440604e45c2926bdb301209
-  data.tar.gz: fda7418c3101bd3e691bd6cb0af7b3b2a6b3c0cd58da30d7a2171868251468c4
+  metadata.gz: 48d47d401250750c0ffa68ccf5f58602a1e0fa4918b0b812aebcd0c60cdfe484
+  data.tar.gz: 9acbfd9d4465c25ddb20186fe57a2d0a84ba74ba24fa2e55b2b2103b5a4d12f9
 SHA512:
-  metadata.gz: 0b10ef50e80806ca90386c4da6ae9057e044af58a1975d3c8b4c03942966b09e23386a64d233916610066ded7c7ec59700ddb4b149107cee71acb6a52459d139
-  data.tar.gz: a977b070175607fed8a2ec30a331278a6f0cd1f6438c1a3b4accffd2be18f8963f92afac01d0144f0567aabefab19c3ed4af4dd4f83c338a3af9bb22022ea542
+  metadata.gz: 7690a2497203522b7c9469656a5e1ea9fd2e72671555548a1d0833841d8b3bce34a60c22b2e9597b94add35342dbae0b2cb3fd58fdb3ec08b9e16375f1ca6044
+  data.tar.gz: 27d5aef80aa2bae48091bfaf398fae956e7226384414bac4505af3752090a722c1be1b421d3260773dd40d7d7c2dc65281b5555bbade267f83554d267371cd85

data/README.md

@@ -13,7 +13,8 @@ Miteru is an experimental phishing kit detection tool.
   - [urlscan.io](https://urlscan.io/search/#certstream-suspicious)
   - [OpenPhish community feed](https://openphish.com/feed.txt)
   - [PhishTank feed](http://data.phishtank.com/data/online-valid.csv)
-- It checks a suspicious URL whether it has a directory listing and contains a phishing kit (`*.zip` file) or not.
+- It checks a suspicious URL whether it has a directory listing and contains a phishing kit (compressed file) or not.
+  - Note: compressed file = `*.zip`, `*.rar`, `*.7z`, `*.tar` and `*.gz`.
 
 ## Installation
 

data/lib/miteru/cli.rb

@@ -8,7 +8,7 @@ require "thor"
 
 module Miteru
   class CLI < Thor
-    method_option :auto_download, type: :boolean, default: false, desc: "Enable or disable auto-download of *.zip file(s)"
+    method_option :auto_download, type: :boolean, default: false, desc: "Enable or disable auto-download of compressed file(s)"
     method_option :directory_traveling, type: :boolean, default: false, desc: "Enable or disable directory traveling"
     method_option :download_to, type: :string, default: "/tmp", desc: "Directory to download file(s)"
     method_option :post_to_slack, type: :boolean, default: false, desc: "Post a message to Slack if it detects a phishing kit"
@@ -26,16 +26,16 @@ module Miteru
       websites.each do |website|
         next unless website.has_kit?
 
-        message = "#{website.url}: it might contain phishing kit(s) (#{website.zip_files.join(', ')})."
+        message = "#{website.url}: it might contain phishing kit(s) (#{website.compressed_files.join(', ')})."
         puts message.colorize(:light_red)
-        post_to_slack(message) if options[:post_to_slack] && valid_slack_setting?
-        download_zip_files(website.url, website.zip_files, options[:download_to]) if options[:auto_download]
+        post_to_slack(website.message) if options[:post_to_slack] && valid_slack_setting?
+        download_compressed_files(website.url, website.compressed_files, options[:download_to]) if options[:auto_download]
       end
     end
 
     no_commands do
-      def download_zip_files(url, zip_files, base_dir)
-        zip_files.each do |path|
+      def download_compressed_files(url, compressed_files, base_dir)
+        compressed_files.each do |path|
           target_url = "#{url}/#{path}"
           begin
             download_file_path = HTTPClient.download(target_url, base_dir)

data/lib/miteru/crawler.rb

@@ -29,17 +29,23 @@ module Miteru
       url = "#{URLSCAN_ENDPOINT}/search/?q=certstream-suspicious&size=#{size}"
       res = JSON.parse(get(url))
       res["results"].map { |result| result.dig("task", "url") }
+    rescue HTTPResponseError => _
+      []
     end
 
     def openphish_feed
       res = get("#{OPENPHISH_ENDPOINT}/feed.txt")
       res.lines.map(&:chomp)
+    rescue HTTPResponseError => _
+      []
     end
 
     def phishtank_feed
       res = get("#{PHISHTANK_ENDPOINT}/data/online-valid.csv")
       table = CSV.parse(res, headers: true)
       table.map { |row| row["url"] }
+    rescue HTTPResponseError => _
+      []
     end
 
     def breakdown(url)
@@ -62,11 +68,15 @@ module Miteru
     end
 
     def suspicious_urls
-      urls = (urlscan_feed + openphish_feed + phishtank_feed)
-      urls.map { |url| breakdown(url) }.flatten.uniq.sort
+      @suspicious_urls ||= [].tap do |arr|
+        urls = (urlscan_feed + openphish_feed + phishtank_feed)
+        urls.map { |url| breakdown(url) }.flatten.uniq.sort.each { |url| arr << url }
+      end
     end
 
     def execute
+      puts "Loaded #{suspicious_urls.length} URLs to crawl." if verbose
+
       pool = Thread.pool(threads)
       websites = []
 

data/lib/miteru/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miteru
-  VERSION = "0.8.0"
+  VERSION = "0.9.0"
 end

data/lib/miteru/website.rb

@@ -14,10 +14,10 @@ module Miteru
       doc.at_css("title")&.text
     end
 
-    def zip_files
-      @zip_files ||= doc.css("a").map do |a|
+    def compressed_files
+      @compressed_files ||= doc.css("a").map do |a|
         href = a.get("href")
-        href&.end_with?(".zip") ? href : nil
+        [".zip", ".rar", ".7z", ".tar", ".gz"].any? { |ext| href&.end_with? ext } ? href : nil
       end.compact.map do |href|
         href.start_with?("/") ? href[1..-1] : href
       end
@@ -31,12 +31,12 @@ module Miteru
       title == "Index of /"
     end
 
-    def zip_files?
-      !zip_files.empty?
+    def compressed_files?
+      !compressed_files.empty?
     end
 
     def has_kit?
-      @has_kit ||= ok? && index? && zip_files?
+      @has_kit ||= ok? && index? && compressed_files?
     end
 
     def build
@@ -46,7 +46,7 @@ module Miteru
     def unbuild
       @doc = nil
       @response = nil
-      @zip_files = nil
+      @compressed_files = nil
     end
 
     private

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miteru
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.9.0
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-10-06 00:00:00.000000000 Z
+date: 2018-10-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler