miteru 0.12.0 → 0.12.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c8e78ff07e29bdfb439c4f5a1c9918fdd689c3ee9a6e076cd48fc08ce2c7f129
-  data.tar.gz: 8524c293c6b72bf6a7d980306fcb15df6053e923241b1403590bb065ee10eac2
+  metadata.gz: c06dbaa1d46bc20c70d44bf5a3fda286e45a9f5bdb5046e9c900e3c6aa5e86a5
+  data.tar.gz: c512cddc5e3c53875e9cc0c0c7db5b61e45fb301ce7cac2a0ab5b46167729df4
 SHA512:
-  metadata.gz: 541a1464c16e6b1182bbc042de1225e7ac1aa04d6b048fe39aaa647f7d59d0ba88feb462f242d244eebb9999f51f1fc2d2d3561081be0a02d512668523eeab40
-  data.tar.gz: 128d70f6b65351a3023d9f9adb34b1bc8ffbcd6bafc9375891d2b8f7817f1da22ee8f367ee0eaf3dff42e29c9b4d381cab118216d53e542da22377029a8dc48d
+  metadata.gz: 7321f78b8d68ce7434f9aaf29b4d781b0cac49ccf48b6a4f9944e6154c55ad0185bc7fe91625a46d7dece574f121d41523df63e761efa264fd4866370c85ccbd
+  data.tar.gz: 4589f29b656ecec1fe9cc03cad0395b49b5f0a9bf674fb36bbcf13b4b2ebfa6cb0601f1d80ea1b94a2fd9f5d33f23b3654c6bc5852a810977da93dcf39f6a2f6

data/README.md

@@ -10,7 +10,7 @@ Miteru is an experimental phishing kit detection tool.
 ## How it works
 
 - It collects phishy URLs from the following feeds:
-  - [urlscan.io certstream-suspicious feed](https://urlscan.io/search/#certstream-suspicious)
+  - [CertStream-Suspicious feed via urlscan.io](https://urlscan.io/search/#certstream-suspicious)
   - [OpenPhish feed via urlscan.io](https://urlscan.io/search/#OpenPhish)
   - [PhishTank feed via urlscan.io](https://urlscan.io/search/#PhishTank)
   - [Ayashige feed](https://github.com/ninoseki/ayashige)
@@ -20,7 +20,7 @@ Miteru is an experimental phishing kit detection tool.
 ## Features
 
 - [x] Phishing kit detection & collection.
-- [x] Slack integration. (Posting a message to Slack if the tool detects a phishing kit.)
+- [x] Slack notification.
 - [x] Threading.
 
 ## Installation

data/lib/miteru.rb

@@ -4,6 +4,7 @@ require "miteru/version"
 
 require "miteru/error"
 require "miteru/http_client"
+require "miteru/kit"
 require "miteru/website"
 require "miteru/downloader"
 require "miteru/feeds"

data/lib/miteru/cli.rb

@@ -4,7 +4,7 @@ require "thor"
 
 module Miteru
   class CLI < Thor
-    method_option :auto_download, type: :boolean, default: false, desc: "Enable or disable auto-download of compressed file(s)"
+    method_option :auto_download, type: :boolean, default: false, desc: "Enable or disable auto-download of phishing kits"
     method_option :directory_traveling, type: :boolean, default: false, desc: "Enable or disable directory traveling"
     method_option :download_to, type: :string, default: "/tmp", desc: "Directory to download file(s)"
     method_option :post_to_slack, type: :boolean, default: false, desc: "Post a message to Slack if it detects a phishing kit"

data/lib/miteru/crawler.rb

@@ -31,12 +31,8 @@ module Miteru
 
       Parallel.each(feeds.suspicious_urls, in_threads: threads) do |url|
         website = Website.new(url)
-        if website.has_kit?
-          downloader.download_compressed_files(website.url, website.compressed_files) if auto_download?
-          notify(website.url, website.compressed_files)
-        else
-          notify(website.url, website.compressed_files) if verbose
-        end
+        downloader.download_kits(website.kits) if website.has_kits? && auto_download?
+        notify(website) if verbose || website.has_kits?
       rescue OpenSSL::SSL::SSLError, HTTP::Error, LL::ParserError, Addressable::URI::InvalidURIError => _
         next
       end
@@ -54,8 +50,8 @@ module Miteru
       ).execute
     end
 
-    def notify(url, message)
-      @notifier.notify(url, message)
+    def notify(website)
+      @notifier.notify(url: website.url, kits: website.kits, message: website.message)
     end
 
     def auto_download?

data/lib/miteru/downloader.rb

@@ -13,33 +13,30 @@ module Miteru
       raise ArgumentError, "#{base_dir} is not existing." unless Dir.exist?(base_dir)
     end
 
-    def download_compressed_files(url, compressed_files)
-      compressed_files.each do |path|
-        target_url = "#{url}/#{path}"
-        filename = download_filename(target_url)
+    def download_kits(kits)
+      kits.each do |kit|
+        filename = download_filename(kit)
         destination = filepath_to_download(filename)
         begin
-          download_filepath = HTTPClient.download(target_url, destination)
-          if duplicated?(download_filepath)
-            puts "Do not download #{target_url} because there is a file that has a same hash value in the directory (SHA256: #{sha256(download_filepath)})."
-            FileUtils.rm download_filepath
+          downloaded_filepath = HTTPClient.download(kit.url, destination)
+          if duplicated?(downloaded_filepath)
+            puts "Do not download #{kit.url} because there is a file that has a same hash value in the directory (SHA256: #{sha256(downloaded_filepath)})."
+            FileUtils.rm downloaded_filepath
           else
-            puts "Download #{target_url} as #{download_filepath}"
+            puts "Download #{kit.url} as #{downloaded_filepath}"
           end
         rescue Down::Error => e
-          puts "Failed to download: #{target_url} (#{e})"
+          puts "Failed to download: #{kit.url} (#{e})"
         end
       end
     end
 
     private
 
-    def download_filename(url)
-      filename = url.split("/").last
-      extname = File.extname(filename)
-      domain = URI(url).hostname
+    def download_filename(kit)
+      domain = URI(kit.base_url).hostname
 
-      "#{domain}_#{filename}_#{SecureRandom.alphanumeric(10)}#{extname}"
+      "#{domain}_#{kit.basename}_#{SecureRandom.alphanumeric(10)}#{kit.extname}"
     end
 
     def filepath_to_download(filename)

data/lib/miteru/kit.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Miteru
+  class Kit
+    VALID_EXTENSIONS = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
+
+    attr_reader :base_url, :link
+
+    def initialize(base_url:, link:)
+      @base_url = base_url
+      @link = link.start_with?("/") ? link[1..-1] : link
+    end
+
+    def valid?
+      VALID_EXTENSIONS.include? extname
+    end
+
+    def extname
+      File.extname(link)
+    end
+
+    def basename
+      File.basename(link)
+    end
+
+    def url
+      "#{base_url}/#{basename}"
+    end
+  end
+end

data/lib/miteru/notifier.rb

@@ -9,11 +9,10 @@ module Miteru
       @post_to_slack = post_to_slack
     end
 
-    def notify(url, compressed_files)
-      message = compressed_files.empty? ? "it doesn't contain a phishing kit." : "it might contain phishing kit(s): (#{compressed_files.join(', ')})."
+    def notify(url:, kits:, message:)
       attachement = Attachement.new(url)
 
-      if post_to_slack? && !compressed_files.empty?
+      if post_to_slack? && !kits.empty?
         slack = Slack::Incoming::Webhooks.new(slack_webhook_url, channel: slack_channel)
         slack.post(
           url,
@@ -24,7 +23,7 @@ module Miteru
         )
       end
 
-      message = message.colorize(:light_red) unless compressed_files.empty?
+      message = message.colorize(:light_red) unless kits.empty?
       puts "#{url}: #{message}"
     end
 

data/lib/miteru/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miteru
-  VERSION = "0.12.0"
+  VERSION = "0.12.1"
 end

data/lib/miteru/website.rb

@@ -13,13 +13,12 @@ module Miteru
       doc.at_css("title")&.text
     end
 
-    def compressed_files
-      @compressed_files ||= doc.css("a").map do |a|
-        href = a.get("href")
-        [".zip", ".rar", ".7z", ".tar", ".gz"].any? { |ext| href&.end_with? ext } ? href : nil
-      end.compact.map do |href|
-        href.start_with?("/") ? href[1..-1] : href
-      end
+    def kits
+      @kits ||= doc.css("a").map do |a|
+        link = a.get("href")
+        kit = Kit.new(base_url: url, link: link.to_s)
+        kit.valid? ? kit : nil
+      end.compact
     end
 
     def ok?
@@ -30,16 +29,21 @@ module Miteru
       title == "Index of /"
     end
 
-    def compressed_files?
-      !compressed_files.empty?
+    def kits?
+      !kits.empty?
     end
 
-    def has_kit?
-      ok? && index? && compressed_files?
-    rescue StandardError => _
+    def has_kits?
+      ok? && index? && kits?
+    rescue OpenSSL::SSL::SSLError, HTTP::Error, LL::ParserError, Addressable::URI::InvalidURIError => _
       false
     end
 
+    def message
+      kit_names = kits.map(&:basename).join(", ")
+      kits? ? "it might contain phishing kit(s): (#{kit_names})." : "it doesn't contain a phishing kit."
+    end
+
     private
 
     def response

data/miteru.gemspec

@@ -36,7 +36,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "down", "~> 4.8"
   spec.add_dependency "http", "~> 4.1"
   spec.add_dependency "oga", "~> 2.15"
-  spec.add_dependency "parallel", "~> 1.14"
+  spec.add_dependency "parallel", "~> 1.16"
   spec.add_dependency "slack-incoming-webhooks", "~> 0.2"
   spec.add_dependency "thor", "~> 0.19"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miteru
 version: !ruby/object:Gem::Version
-  version: 0.12.0
+  version: 0.12.1
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-03-17 00:00:00.000000000 Z
+date: 2019-03-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -170,14 +170,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.14'
+        version: '1.16'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.14'
+        version: '1.16'
 - !ruby/object:Gem::Dependency
   name: slack-incoming-webhooks
   requirement: !ruby/object:Gem::Requirement
@@ -236,6 +236,7 @@ files:
 - lib/miteru/feeds/feed.rb
 - lib/miteru/feeds/urlscan.rb
 - lib/miteru/http_client.rb
+- lib/miteru/kit.rb
 - lib/miteru/notifier.rb
 - lib/miteru/version.rb
 - lib/miteru/website.rb