miteru 0.12.12 → 0.12.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5aad882823cf7ae42d80c805f109dcd05d4ec483c7a40354c19d1c2fd17d6466
-  data.tar.gz: 96bdfc368fe931d1c90a77496ef02491f2697037324d9a38e9b8bbb626c9f8f7
+  metadata.gz: feb9c62c511e26e85a86755680ed77ddb34aca15c6d17d8c57dd2c60aa84becf
+  data.tar.gz: d9e86a727e12e171b65accb829b07602847986ba81ea6457400ad8527b862324
 SHA512:
-  metadata.gz: 11097b5429402c9123404ee3aa02544867d44507dfaf87ca4a09179cc8caf02846f57b6fdc8e9e3c8a65ba90f95b57bdebc7b208c2503e9564c7d3a20439f22e
-  data.tar.gz: eb9b0f9eb1d05af88e4c520eb1edf64807f3c57e9abb94f0ee0956f68951d87bd18213d11cbe9f83a1150333096556638f9a0352e050bf15fc913a1b77ac7548
+  metadata.gz: 8402163c1decbcee1c352a2ca9004f52b8c0262c5adc18d10866abe190151c8bf56d1e7ed03018f33539d1f108a1ccfc265d289dcaeb167606a10217ec112d6a
+  data.tar.gz: 6ecde10d8a4d1842995a2281e1b30663cb25c3fdd9edefa5c8f883c94f2da3b973372a84adfa70bd8537c126ab12b870b2bf6046f3021e329129428974fc96c0

data/README.md

@@ -2,7 +2,8 @@
 
 [![Gem Version](https://badge.fury.io/rb/miteru.svg)](https://badge.fury.io/rb/miteru)
 [![Build Status](https://travis-ci.com/ninoseki/miteru.svg?branch=master)](https://travis-ci.com/ninoseki/miteru)
-[![Maintainability](https://api.codeclimate.com/v1/badges/d90e1b5bbdd9663a17d1/maintainability)](https://codeclimate.com/github/ninoseki/miteru/maintainability)
+![Docker Cloud Build Status](https://img.shields.io/docker/cloud/build/ninoseki/miteru)
+[![CodeFactor](https://www.codefactor.io/repository/github/ninoseki/miteru/badge)](https://www.codefactor.io/repository/github/ninoseki/miteru)
 [![Coverage Status](https://coveralls.io/repos/github/ninoseki/miteru/badge.svg?branch=master)](https://coveralls.io/github/ninoseki/miteru?branch=master)
 
 Miteru is an experimental phishing kit detection tool.
@@ -13,6 +14,7 @@ Miteru is an experimental phishing kit detection tool.
   - [CertStream-Suspicious feed via urlscan.io](https://urlscan.io/search/#certstream-suspicious)
   - [OpenPhish feed via urlscan.io](https://urlscan.io/search/#OpenPhish)
   - [PhishTank feed via urlscan.io](https://urlscan.io/search/#PhishTank)
+  - [URLhaus feed via urlscan.io](https://urlscan.io/search/#URLHaus)
   - [Ayashige feed](https://github.com/ninoseki/ayashige)
 - It checks each phishy URL whether it enables directory listing and contains a phishing kit (compressed file) or not.
   - Note: compressed file = `*.zip`, `*.rar`, `*.7z`, `*.tar` and `*.gz`.

data/lib/miteru/crawler.rb

@@ -11,7 +11,6 @@ module Miteru
 
     def initialize
       @downloader = Downloader.new(Miteru.configuration.download_to)
-
       @feeds = Feeds.new
       @notifier = Notifier.new
     end
@@ -25,7 +24,6 @@ module Miteru
     end
 
     def execute
-      threads = Miteru.configuration.threads
       suspicious_urls = feeds.suspicious_urls
       puts "Loaded #{suspicious_urls.length} URLs to crawl. (crawling in #{threads} threads)" if verbose?
 
@@ -34,8 +32,8 @@ module Miteru
       end
     end
 
-    def self.execute
-      new.execute
+    def threads
+      @threads ||= Miteru.configuration.threads
     end
 
     def notify(website)
@@ -49,5 +47,11 @@ module Miteru
     def verbose?
       Miteru.configuration.verbose?
     end
+
+    class << self
+      def execute
+        new.execute
+      end
+    end
   end
 end

data/lib/miteru/downloader.rb

@@ -22,8 +22,7 @@ module Miteru
     private
 
     def download_kit(kit)
-      filename = download_filename(kit)
-      destination = filepath_to_download(filename)
+      destination = kit.download_filepath
       begin
         downloaded_filepath = HTTPClient.download(kit.url, destination)
         hash = sha256(downloaded_filepath)
@@ -38,12 +37,6 @@ module Miteru
       end
     end
 
-    def download_filename(kit)
-      domain = URI(kit.base_url).hostname
-
-      "#{domain}_#{kit.filename}_#{SecureRandom.alphanumeric(10)}#{kit.extname}"
-    end
-
     def filepath_to_download(filename)
       "#{base_dir}/#{filename}"
     end

data/lib/miteru/feeds.rb

@@ -6,9 +6,13 @@ require_relative "./feeds/urlscan"
 
 module Miteru
   class Feeds
+    IGNORE_EXTENSIONS = %w(.htm .html .php .asp .aspx .exe .txt).freeze
+
     def initialize
-      @feeds = [UrlScan.new(Miteru.configuration.size)]
-      @feeds << Ayashige.new if Miteru.configuration.ayashige?
+      @feeds = [
+        UrlScan.new(Miteru.configuration.size),
+        Miteru.configuration.ayashige? ? Ayashige.new : nil
+      ].compact
     end
 
     def directory_traveling?
@@ -41,8 +45,12 @@ module Miteru
       urls = (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join('/')}" }
       urls.reject do |breakdowned_url|
         # Reject a url which ends with specific extension names
-        %w(.htm .html .php .asp .aspx).any? { |ext| breakdowned_url.end_with? ext }
+        invalid_extension? breakdowned_url
       end
     end
+
+    def invalid_extension?(url)
+      IGNORE_EXTENSIONS.any? { |ext| url.end_with? ext }
+    end
   end
 end

data/lib/miteru/http_client.rb

@@ -2,7 +2,6 @@
 
 require "down/http"
 require "http"
-require "securerandom"
 require "uri"
 
 module Miteru

data/lib/miteru/kit.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "cgi"
+require "securerandom"
 
 module Miteru
   class Kit
@@ -34,5 +35,39 @@ module Miteru
     def url
       "#{base_url}/#{basename}"
     end
+
+    def download_filepath
+      "#{base_dir}/#{download_filename}"
+    end
+
+    def filesize
+      return nil unless File.exist?(download_filepath)
+
+      File.size download_filepath
+    end
+
+    def filename_with_size
+      return filename unless filesize
+
+      "#{filename}(#{filesize / 1024}KB)"
+    end
+
+    private
+
+    def id
+      @id ||= SecureRandom.hex(10)
+    end
+
+    def hostname
+      URI(base_url).hostname
+    end
+
+    def download_filename
+      "#{hostname}_#{filename}_#{id}#{extname}"
+    end
+
+    def base_dir
+      @base_dir ||= Miteru.configuration.download_to
+    end
   end
 end

data/lib/miteru/notifier.rb

@@ -7,13 +7,14 @@ module Miteru
   class Notifier
     def notify(url:, kits:, message:)
       attachement = Attachement.new(url)
+      kits = kits.select(&:filesize)
 
-      if post_to_slack? && !kits.empty?
+      if post_to_slack? && kits.any?
         notifier = Slack::Notifier.new(slack_webhook_url, channel: slack_channel)
         notifier.post(text: message, attachments: attachement.to_a)
       end
 
-      message = message.colorize(:light_red) unless kits.empty?
+      message = message.colorize(:light_red) if kits.any?
       puts "#{url}: #{message}"
     end
 

data/lib/miteru/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miteru
-  VERSION = "0.12.12"
+  VERSION = "0.12.13"
 end

data/lib/miteru/website.rb

@@ -41,9 +41,9 @@ module Miteru
     def message
       return "It doesn't contain a phishing kit." unless kits?
 
-      kit_names = kits.map(&:filename).join(", ")
+      filename_with_sizes = kits.map(&:filename_with_size).join(", ")
       noun = kits.length == 1 ? "a phishing kit" : "phishing kits"
-      "It might contain #{noun}: #{kit_names}."
+      "It might contain #{noun}: #{filename_with_sizes}."
     end
 
     private

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miteru
 version: !ruby/object:Gem::Version
-  version: 0.12.12
+  version: 0.12.13
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-11-26 00:00:00.000000000 Z
+date: 2019-11-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler