miteru 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aa876b402ad3fcfe44fb1a66035bdb7649db11383650faf572d94fb8b4c8f498
-  data.tar.gz: c8ffde9ad933fbe670900d8b6547f5ce35ba7f85bd0a0d1201d3c00518d612bb
+  metadata.gz: b3784652417f9e36941b1bc9ed8a76acaa86481a4730223b8ad1913775980097
+  data.tar.gz: 47526f0cc44c07a8aa9a651973f0fa4a3d455dfe49abed255f0aed1780abb604
 SHA512:
-  metadata.gz: 15bc48dc53696348635d637674c5574d537b8cf220939dce354875706f1a58820b95ecab547da0106a72e76d026d3e067b14438bfbef7ce1ce971737491269e2
-  data.tar.gz: 86d0aad06740b2013c35ef3920fa27fc1ac811580c08103273f3c073362d37b0e2c8bd0e49c9d8aac63dd06004556b902c6bccaf57f7833670e01d8e907d72a1
+  metadata.gz: 124a5071040ee603dab9e5986064566e6e61d2acb812d052bc8cee69a673253894004b1824c3fac80fa467674640c2a21edec37a14aa1b22d1d03d40c7b99b4c
+  data.tar.gz: 7da8680a710bef8081b2ba279843b53a28fba36c29290ba9566ce3e40fd892ac714a92feec4008236d2b84246ac5bbaa3d94600d008b39953c6103630faa3627

data/README.md

@@ -9,8 +9,8 @@ Miteru is an experimental phishing kit detection tool.
 
 ## How it works
 
-- It collects phishing suspicious URLs from [urlscan.io](https://urlscan.io/search/#certstream-suspicious).
-- It checks a suspicious URL whether it contains a phishing kit (`*.zip` file) or not.
+- It collects phishing suspicious URLs from [urlscan.io](https://urlscan.io/search/#certstream-suspicious) and [OpenPhish community feed](https://openphish.com/feed.txt).
+- It checks a suspicious URL whether it has a directory listing and contains a phishing kit (`*.zip` file) or not.
 
 ## Installation
 

data/lib/miteru/cli.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require "colorize"
+require "digest"
+require "fileutils"
 require "http"
 require "thor"
 
@@ -17,7 +19,7 @@ module Miteru
       websites.each do |website|
         next unless website.has_kit?
 
-        message = "#{website.url}: it might contain a phishing kit (#{website.zip_files.join(',')})."
+        message = "#{website.url}: it might contain phishing kit(s) (#{website.zip_files.join(', ')})."
         puts message.colorize(:light_red)
         post_to_slack(message) if options[:post_to_slack] && valid_slack_setting?
         download_zip_files(website.url, website.zip_files, options[:download_to]) if options[:auto_download]
@@ -29,14 +31,30 @@ module Miteru
         zip_files.each do |path|
           target_url = "#{url}/#{path}"
           begin
-            destination = HTTPClient.download(target_url, base_dir)
-            puts "Download #{target_url} as #{destination}"
+            download_file_path = HTTPClient.download(target_url, base_dir)
+            if duplicated?(download_file_path, base_dir)
+              puts "Do not download #{target_url} because there is a same hash file in the directory (SHA256: #{sha256(download_file_path)})."
+              FileUtils.rm download_file_path
+            else
+              puts "Download #{target_url} as #{download_file_path}"
+            end
           rescue Down::Error => e
             puts "Failed to download: #{target_url} (#{e})"
           end
         end
       end
 
+      def sha256(path)
+        digest = Digest::SHA256.file(path)
+        digest.hexdigest
+      end
+
+      def duplicated?(file_path, base_dir)
+        base = sha256(file_path)
+        sha256s = Dir.glob("#{base_dir}/*.zip").map { |path| sha256(path) }
+        sha256s.select { |sha256| sha256 == base }.length > 1
+      end
+
       def valid_slack_setting?
         ENV["SLACK_WEBHOOK_URL"] != nil
       end

data/lib/miteru/crawler.rb

@@ -1,7 +1,9 @@
 # frozen_string_literal: true
 
-require "thread/pool"
 require "http"
+require "json"
+require "thread/pool"
+require "uri"
 
 module Miteru
   class Crawler
@@ -9,6 +11,9 @@ module Miteru
     attr_reader :size
     attr_reader :verbose
 
+    URLSCAN_ENDPOINT = "https://urlscan.io/api/v1"
+    OPENPHISH_ENDPOINT = "https://openphish.com"
+
     def initialize(size: 100, verbose: false)
       @threads = 10
       @size = size
@@ -16,17 +21,44 @@ module Miteru
       raise ArgumentError, "size must be less than 100,000" if size > 100_000
     end
 
-    def suspicous_urls
-      url = "https://urlscan.io/api/v1/search/?q=certstream-suspicious&size=#{size}"
+    def urlscan_feed
+      url = "#{URLSCAN_ENDPOINT}/search/?q=certstream-suspicious&size=#{size}"
       res = JSON.parse(get(url))
       res["results"].map { |result| result.dig("task", "url") }
     end
 
+    def openphish_feed
+      res = get("#{OPENPHISH_ENDPOINT}/feed.txt")
+      res.lines.map(&:chomp)
+    end
+
+    def breakdown(url)
+      begin
+        uri = URI.parse(url)
+      rescue URI::InvalidURIError => _
+        return []
+      end
+      base = "#{uri.scheme}://#{uri.hostname}"
+      [base]
+      # TODO: Should add a option for burute force directory
+      # segments = uri.path.split("/")
+      # if segments.length.zero?
+      #   [base]
+      # else
+      #   (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join('/')}" }
+      # end
+    end
+
+    def suspicious_urls
+      urls = urlscan_feed + openphish_feed
+      urls.map { |url| breakdown(url) }.flatten.uniq.sort
+    end
+
     def execute
       pool = Thread.pool(threads)
       websites = []
 
-      suspicous_urls.each do |url|
+      suspicious_urls.each do |url|
         pool.process do
           website = Website.new(url)
           puts "#{website.url}: it doesn't contain a phishing kit." if verbose && !website.has_kit?

data/lib/miteru/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miteru
-  VERSION = "0.4.0"
+  VERSION = "0.5.0"
 end

data/lib/miteru/website.rb

@@ -18,7 +18,9 @@ module Miteru
       @zip_files ||= doc.css("a").map do |a|
         href = a.get("href")
         href&.end_with?(".zip") ? href : nil
-      end.compact
+      end.compact.map do |href|
+        href.start_with?("/") ? href[1..-1] : href
+      end
     end
 
     def ok?

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miteru
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-09-22 00:00:00.000000000 Z
+date: 2018-09-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler