miteru 0.12.11 → 0.14.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 80ce8c8350165a57da77e2f4c7e86acd36e48b921a114fc8299ac7925847b696
-  data.tar.gz: 5c1efd6135cef875a7d425ee5f22c65553b05564afcff2b6ccc8234990db0fc2
+  metadata.gz: cbb47828d938e86289f845435bd9d342e18d7c50090e486b5681bf212dfcd84b
+  data.tar.gz: c416439bc1c80cfd4cf2de8bc9a7a6abb6ff46bb1af5d0a58ec0e71954584a4c
 SHA512:
-  metadata.gz: 7bcf546a3dedf9cc4e9b1bc5aeb52806846fba4828279fc75afcf6c113c9ddf87646ecd2c737f8078c77ef391e13d56aafea00eba00073884cd2ada0acfd11b2
-  data.tar.gz: 220a3b82b01ec336d98217edb2a8940bab3ba46ab786aa5ed34aec93a23d55fadd9fb4a1668ca96f90f0c2f1679b8048d3b418970f2d79ee4819df779f49607d
+  metadata.gz: f15f1770d870404e7b50cddb2e20d7e5d90adfab04283d5cb4364a2480f210b9c4869717d647bdc0174b5d0f0a9323903aac101f77d140ebad0c9523688b43d0
+  data.tar.gz: cb4a8e2c561aa53265362beec7e45445f6a6825bf991c7c4c559d40279dc84e138f54e300ab163f9db19dc891f2fe0a30c874d620d0464c9451c1e6c0e6e729e

data/.travis.yml

@@ -2,8 +2,6 @@ sudo: false
 language: ruby
 cache: bundler
 rvm:
-  - 2.5.1
-  - 2.6.1
-before_install:
-  - gem update --system
-  - gem install bundler
+  - 2.6
+  - 2.7
+before_install: gem install bundler -v 2.1

data/README.md

@@ -1,8 +1,9 @@
 # Miteru
 
 [![Gem Version](https://badge.fury.io/rb/miteru.svg)](https://badge.fury.io/rb/miteru)
-[![Build Status](https://travis-ci.org/ninoseki/miteru.svg?branch=master)](https://travis-ci.org/ninoseki/miteru)
-[![Maintainability](https://api.codeclimate.com/v1/badges/d90e1b5bbdd9663a17d1/maintainability)](https://codeclimate.com/github/ninoseki/miteru/maintainability)
+[![Build Status](https://travis-ci.com/ninoseki/miteru.svg?branch=master)](https://travis-ci.com/ninoseki/miteru)
+[![Docker Cloud Build Status](https://img.shields.io/docker/cloud/build/ninoseki/miteru)](https://hub.docker.com/repository/docker/ninoseki/miteru)
+[![CodeFactor](https://www.codefactor.io/repository/github/ninoseki/miteru/badge)](https://www.codefactor.io/repository/github/ninoseki/miteru)
 [![Coverage Status](https://coveralls.io/repos/github/ninoseki/miteru/badge.svg?branch=master)](https://coveralls.io/github/ninoseki/miteru?branch=master)
 
 Miteru is an experimental phishing kit detection tool.
@@ -13,7 +14,11 @@ Miteru is an experimental phishing kit detection tool.
   - [CertStream-Suspicious feed via urlscan.io](https://urlscan.io/search/#certstream-suspicious)
   - [OpenPhish feed via urlscan.io](https://urlscan.io/search/#OpenPhish)
   - [PhishTank feed via urlscan.io](https://urlscan.io/search/#PhishTank)
+  - [URLhaus feed via urlscan.io](https://urlscan.io/search/#URLHaus)
+  - urlscan.io phish feed (available for Pro users)
   - [Ayashige feed](https://github.com/ninoseki/ayashige)
+  - [Phishing Database feed](https://github.com/mitchellkrogza/Phishing.Database)
+  - [PhishStats feed](https://phishstats.info/)
 - It checks each phishy URL whether it enables directory listing and contains a phishing kit (compressed file) or not.
   - Note: compressed file = `*.zip`, `*.rar`, `*.7z`, `*.tar` and `*.gz`.
 
@@ -83,6 +88,10 @@ For using `--post-to-slack` feature, you should set the following environment va
 - `SLACK_WEBHOOK_URL`: Your Slack Webhook URL.
 - `SLACK_CHANNEL`: Slack channel to post a message (default: "#general").
 
+If you are a urlscan.io Pro user, set your API key as an environment variable `URLSCAN_API_KEY`.
+
+It enables you to subscribe the urlscan.io phish feed.
+
 ## Examples
 
 ### Aasciinema cast

data/lib/miteru/configuration.rb

@@ -36,7 +36,7 @@ module Miteru
       @post_to_slack = false
       @size = 100
       @threads = Parallel.processor_count
-      @verbaose = false
+      @verbose = false
     end
 
     def auto_download?
@@ -56,7 +56,7 @@ module Miteru
     end
 
     def verbose?
-      @verbaose
+      @verbose
     end
   end
 

data/lib/miteru/crawler.rb

@@ -11,7 +11,6 @@ module Miteru
 
     def initialize
       @downloader = Downloader.new(Miteru.configuration.download_to)
-
       @feeds = Feeds.new
       @notifier = Notifier.new
     end
@@ -25,7 +24,6 @@ module Miteru
     end
 
     def execute
-      threads = Miteru.configuration.threads
       suspicious_urls = feeds.suspicious_urls
       puts "Loaded #{suspicious_urls.length} URLs to crawl. (crawling in #{threads} threads)" if verbose?
 
@@ -34,8 +32,8 @@ module Miteru
       end
     end
 
-    def self.execute
-      new.execute
+    def threads
+      @threads ||= Miteru.configuration.threads
     end
 
     def notify(website)
@@ -49,5 +47,11 @@ module Miteru
     def verbose?
       Miteru.configuration.verbose?
     end
+
+    class << self
+      def execute
+        new.execute
+      end
+    end
   end
 end

data/lib/miteru/downloader.rb

@@ -22,8 +22,7 @@ module Miteru
     private
 
     def download_kit(kit)
-      filename = download_filename(kit)
-      destination = filepath_to_download(filename)
+      destination = kit.download_filepath
       begin
         downloaded_filepath = HTTPClient.download(kit.url, destination)
         hash = sha256(downloaded_filepath)
@@ -38,12 +37,6 @@ module Miteru
       end
     end
 
-    def download_filename(kit)
-      domain = URI(kit.base_url).hostname
-
-      "#{domain}_#{kit.filename}_#{SecureRandom.alphanumeric(10)}#{kit.extname}"
-    end
-
     def filepath_to_download(filename)
       "#{base_dir}/#{filename}"
     end

data/lib/miteru/feeds.rb

@@ -1,14 +1,25 @@
 # frozen_string_literal: true
 
 require_relative "./feeds/feed"
+require_relative "./feeds/phishing_database"
+require_relative "./feeds/phishstats"
 require_relative "./feeds/ayashige"
 require_relative "./feeds/urlscan"
+require_relative "./feeds/urlscan_pro"
 
 module Miteru
   class Feeds
+    IGNORE_EXTENSIONS = %w(.htm .html .php .asp .aspx .exe .txt).freeze
+    VALID_EXTENSIONS = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
+
     def initialize
-      @feeds = [UrlScan.new(Miteru.configuration.size)]
-      @feeds << Ayashige.new if Miteru.configuration.ayashige?
+      @feeds = [
+        PhishingDatabase.new,
+        PhishStats.new,
+        UrlScan.new(Miteru.configuration.size),
+        UrlScanPro.new,
+        Miteru.configuration.ayashige? ? Ayashige.new : nil
+      ].compact
     end
 
     def directory_traveling?
@@ -38,11 +49,25 @@ module Miteru
       segments = uri.path.split("/")
       return [base] if segments.length.zero?
 
-      urls = (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join('/')}" }
+      urls = (0...segments.length).map do |idx|
+        breakdowned_url = "#{base}#{segments[0..idx].join('/')}"
+        breakdown = [breakdowned_url]
+        if idx > 0 && idx < segments.length - 1
+          VALID_EXTENSIONS.each do |ext|
+            breakdown << "#{base}#{segments[0..idx - 1].join('/')}/#{segments[idx]}#{ext}"
+          end
+        end
+        breakdown
+      end.flatten
+
       urls.reject do |breakdowned_url|
         # Reject a url which ends with specific extension names
-        %w(.htm .html .php .asp .aspx).any? { |ext| breakdowned_url.end_with? ext }
+        invalid_extension? breakdowned_url
       end
     end
+
+    def invalid_extension?(url)
+      IGNORE_EXTENSIONS.any? { |ext| url.end_with? ext }
+    end
   end
 end

data/lib/miteru/feeds/phishing_database.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require "json"
+require "uri"
+
+module Miteru
+  class Feeds
+    class PhishingDatabase < Feed
+      URL = "https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/phishing-links-NEW-today.txt"
+
+      def urls
+        body = get(URL)
+        body.to_s.lines.map(&:chomp)
+      rescue HTTPResponseError, HTTP::Error, JSON::ParserError => e
+        puts "Failed to load phishing database feed (#{e})"
+        []
+      end
+    end
+  end
+end

data/lib/miteru/feeds/phishstats.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "json"
+require "uri"
+
+module Miteru
+  class Feeds
+    class PhishStats < Feed
+      URL = "https://phishstats.info:2096/api/phishing?_sort=-id&size=100"
+
+      def urls
+        json = JSON.parse(get(URL))
+        json.map do |entry|
+          entry.dig("url")
+        end
+      rescue HTTPResponseError, HTTP::Error, JSON::ParserError => e
+        puts "Failed to load PhishStats feed (#{e})"
+        []
+      end
+
+      private
+
+      def url_for(path)
+        URI(URL + path)
+      end
+    end
+  end
+end

data/lib/miteru/feeds/urlscan.rb

@@ -1,39 +1,35 @@
 # frozen_string_literal: true
 
-require "json"
-require "uri"
+require "urlscan"
 
 module Miteru
   class Feeds
     class UrlScan < Feed
-      HOST = "urlscan.io"
-      VERSION = 1
-      URL = "https://#{HOST}/api/v#{VERSION}"
-
       attr_reader :size
+
       def initialize(size = 100)
         @size = size
         raise ArgumentError, "size must be less than 10,000" if size > 10_000
       end
 
+      def api
+        @api ||= ::UrlScan::API.new
+      end
+
       def urls
-        url = url_for("/search/")
-        url.query = URI.encode_www_form(
-          q: "PhishTank OR OpenPhish OR CertStream-Suspicious",
-          size: size
-        )
-
-        res = JSON.parse(get(url))
-        res["results"].map { |result| result.dig("task", "url") }
-      rescue HTTPResponseError, HTTP::Error, JSON::ParserError => e
+        urls_from_community_feed
+      rescue ::UrlScan::ResponseError => e
         puts "Failed to load urlscan.io feed (#{e})"
         []
       end
 
       private
 
-      def url_for(path)
-        URI(URL + path)
+      def urls_from_community_feed
+        res = api.search("task.method:automatic", size: size)
+
+        results = res["results"] || []
+        results.map { |result| result.dig("task", "url") }
       end
     end
   end

data/lib/miteru/feeds/urlscan_pro.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require "urlscan"
+
+module Miteru
+  class Feeds
+    class UrlScanPro < Feed
+      def api
+        @api ||= ::UrlScan::API.new
+      end
+
+      def urls
+        urls_from_pro_feed
+      rescue ::UrlScan::ResponseError => e
+        puts "Failed to load urlscan.io pro feed (#{e})"
+        []
+      end
+
+      private
+
+      def api_key?
+        ENV.key? "URLSCAN_API_KEY"
+      end
+
+      def urls_from_pro_feed
+        return [] unless api_key?
+
+        res = api.pro.phishfeed
+        results = res["results"] || []
+        results.map { |result| result.dig("page_url") }
+      rescue ArgumentError => _e
+        []
+      end
+    end
+  end
+end

data/lib/miteru/http_client.rb

@@ -2,7 +2,6 @@
 
 require "down/http"
 require "http"
-require "securerandom"
 require "uri"
 
 module Miteru

data/lib/miteru/kit.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "cgi"
+require "securerandom"
 
 module Miteru
   class Kit
@@ -34,5 +35,39 @@ module Miteru
     def url
       "#{base_url}/#{basename}"
     end
+
+    def download_filepath
+      "#{base_dir}/#{download_filename}"
+    end
+
+    def filesize
+      return nil unless File.exist?(download_filepath)
+
+      File.size download_filepath
+    end
+
+    def filename_with_size
+      return filename unless filesize
+
+      "#{filename}(#{filesize / 1024}KB)"
+    end
+
+    private
+
+    def id
+      @id ||= SecureRandom.hex(10)
+    end
+
+    def hostname
+      URI(base_url).hostname
+    end
+
+    def download_filename
+      "#{hostname}_#{filename}_#{id}#{extname}"
+    end
+
+    def base_dir
+      @base_dir ||= Miteru.configuration.download_to
+    end
   end
 end

data/lib/miteru/notifier.rb

@@ -7,13 +7,14 @@ module Miteru
   class Notifier
     def notify(url:, kits:, message:)
       attachement = Attachement.new(url)
+      kits = kits.select(&:filesize)
 
-      if post_to_slack? && !kits.empty?
+      if post_to_slack? && kits.any?
         notifier = Slack::Notifier.new(slack_webhook_url, channel: slack_channel)
         notifier.post(text: message, attachments: attachement.to_a)
       end
 
-      message = message.colorize(:light_red) unless kits.empty?
+      message = message.colorize(:light_red) if kits.any?
       puts "#{url}: #{message}"
     end
 

data/lib/miteru/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miteru
-  VERSION = "0.12.11"
+  VERSION = "0.14.1"
 end

data/lib/miteru/website.rb

@@ -41,9 +41,9 @@ module Miteru
     def message
       return "It doesn't contain a phishing kit." unless kits?
 
-      kit_names = kits.map(&:filename).join(", ")
+      filename_with_sizes = kits.map(&:filename_with_size).join(", ")
       noun = kits.length == 1 ? "a phishing kit" : "phishing kits"
-      "It might contain #{noun}: #{kit_names}."
+      "It might contain #{noun}: #{filename_with_sizes}."
     end
 
     private

data/miteru.gemspec

@@ -24,19 +24,20 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 2.0"
+  spec.add_development_dependency "bundler", "~> 2.1"
   spec.add_development_dependency "coveralls", "~> 0.8"
   spec.add_development_dependency "glint", "~> 0.1"
   spec.add_development_dependency "rake", "~> 13.0"
-  spec.add_development_dependency "rspec", "~> 3.8"
-  spec.add_development_dependency "vcr", "~> 5.0"
-  spec.add_development_dependency "webmock", "~> 3.7"
+  spec.add_development_dependency "rspec", "~> 3.9"
+  spec.add_development_dependency "vcr", "~> 6.0"
+  spec.add_development_dependency "webmock", "~> 3.8"
 
   spec.add_dependency "colorize", "~> 0.8"
-  spec.add_dependency "down", "~> 5.0"
-  spec.add_dependency "http", "~> 4.1"
-  spec.add_dependency "oga", "~> 2.15"
-  spec.add_dependency "parallel", "~> 1.18"
+  spec.add_dependency "down", "~> 5.1"
+  spec.add_dependency "http", "~> 4.4"
+  spec.add_dependency "oga", "~> 3.2"
+  spec.add_dependency "parallel", "~> 1.19"
   spec.add_dependency "slack-notifier", "~> 2.3"
-  spec.add_dependency "thor", "~> 0.20"
+  spec.add_dependency "thor", "~> 1.0"
+  spec.add_dependency "urlscan", "~> 0.5"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miteru
 version: !ruby/object:Gem::Version
-  version: 0.12.11
+  version: 0.14.1
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-10-09 00:00:00.000000000 Z
+date: 2020-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '2.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '2.1'
 - !ruby/object:Gem::Dependency
   name: coveralls
   requirement: !ruby/object:Gem::Requirement
@@ -72,42 +72,42 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.8'
+        version: '3.9'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.8'
+        version: '3.9'
 - !ruby/object:Gem::Dependency
   name: vcr
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '6.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '6.0'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.7'
+        version: '3.8'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.7'
+        version: '3.8'
 - !ruby/object:Gem::Dependency
   name: colorize
   requirement: !ruby/object:Gem::Requirement
@@ -128,56 +128,56 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '5.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '5.1'
 - !ruby/object:Gem::Dependency
   name: http
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.1'
+        version: '4.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.1'
+        version: '4.4'
 - !ruby/object:Gem::Dependency
   name: oga
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.15'
+        version: '3.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.15'
+        version: '3.2'
 - !ruby/object:Gem::Dependency
   name: parallel
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.18'
+        version: '1.19'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.18'
+        version: '1.19'
 - !ruby/object:Gem::Dependency
   name: slack-notifier
   requirement: !ruby/object:Gem::Requirement
@@ -198,14 +198,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.20'
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: urlscan
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.20'
+        version: '0.5'
 description: An experimental phishing kit detector
 email:
 - manabu.niseki@gmail.com
@@ -235,7 +249,10 @@ files:
 - lib/miteru/feeds.rb
 - lib/miteru/feeds/ayashige.rb
 - lib/miteru/feeds/feed.rb
+- lib/miteru/feeds/phishing_database.rb
+- lib/miteru/feeds/phishstats.rb
 - lib/miteru/feeds/urlscan.rb
+- lib/miteru/feeds/urlscan_pro.rb
 - lib/miteru/http_client.rb
 - lib/miteru/kit.rb
 - lib/miteru/notifier.rb
@@ -262,7 +279,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: An experimental phishing kit detector