miteru 0.12.13 → 0.14.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: feb9c62c511e26e85a86755680ed77ddb34aca15c6d17d8c57dd2c60aa84becf
-  data.tar.gz: d9e86a727e12e171b65accb829b07602847986ba81ea6457400ad8527b862324
+  metadata.gz: 95bccae4b2e34a97963be17cea1b90180ab18278b07be56db5bbc9e1755f673a
+  data.tar.gz: 0cf9b0015a178f037c802cd88da68828d6db7618cd35a4fc4e9a6dd9f5030703
 SHA512:
-  metadata.gz: 8402163c1decbcee1c352a2ca9004f52b8c0262c5adc18d10866abe190151c8bf56d1e7ed03018f33539d1f108a1ccfc265d289dcaeb167606a10217ec112d6a
-  data.tar.gz: 6ecde10d8a4d1842995a2281e1b30663cb25c3fdd9edefa5c8f883c94f2da3b973372a84adfa70bd8537c126ab12b870b2bf6046f3021e329129428974fc96c0
+  metadata.gz: 65ebef82c94e969c4886a34bab6241aea8c6a495a041a10a8df7af7429d77a0eda03f233df8ae56b904542d649c8a9bb549e6ceeb4007b5f19ad2616407b20ff
+  data.tar.gz: 2d145f705bd18c0219695ae83aeff01bccfeb3fc34dc04a0a9dcf9374a9b839bd43762d76501430509baac8598895df088e35dcd24a5c846e5962072ce00ec40

data/.travis.yml

@@ -3,6 +3,5 @@ language: ruby
 cache: bundler
 rvm:
   - 2.6
-before_install:
-  - gem update --system
-  - gem install bundler
+  - 2.7
+before_install: gem install bundler -v 2.1

data/README.md

@@ -2,7 +2,7 @@
 
 [![Gem Version](https://badge.fury.io/rb/miteru.svg)](https://badge.fury.io/rb/miteru)
 [![Build Status](https://travis-ci.com/ninoseki/miteru.svg?branch=master)](https://travis-ci.com/ninoseki/miteru)
-![Docker Cloud Build Status](https://img.shields.io/docker/cloud/build/ninoseki/miteru)
+[![Docker Cloud Build Status](https://img.shields.io/docker/cloud/build/ninoseki/miteru)](https://hub.docker.com/repository/docker/ninoseki/miteru)
 [![CodeFactor](https://www.codefactor.io/repository/github/ninoseki/miteru/badge)](https://www.codefactor.io/repository/github/ninoseki/miteru)
 [![Coverage Status](https://coveralls.io/repos/github/ninoseki/miteru/badge.svg?branch=master)](https://coveralls.io/github/ninoseki/miteru?branch=master)
 
@@ -15,7 +15,10 @@ Miteru is an experimental phishing kit detection tool.
   - [OpenPhish feed via urlscan.io](https://urlscan.io/search/#OpenPhish)
   - [PhishTank feed via urlscan.io](https://urlscan.io/search/#PhishTank)
   - [URLhaus feed via urlscan.io](https://urlscan.io/search/#URLHaus)
+  - urlscan.io phish feed (available for Pro users)
   - [Ayashige feed](https://github.com/ninoseki/ayashige)
+  - [Phishing Database feed](https://github.com/mitchellkrogza/Phishing.Database)
+  - [PhishStats feed](https://phishstats.info/)
 - It checks each phishy URL whether it enables directory listing and contains a phishing kit (compressed file) or not.
   - Note: compressed file = `*.zip`, `*.rar`, `*.7z`, `*.tar` and `*.gz`.
 
@@ -85,6 +88,10 @@ For using `--post-to-slack` feature, you should set the following environment va
 - `SLACK_WEBHOOK_URL`: Your Slack Webhook URL.
 - `SLACK_CHANNEL`: Slack channel to post a message (default: "#general").
 
+If you are a urlscan.io Pro user, set your API key as an environment variable `URLSCAN_API_KEY`.
+
+It enables you to subscribe the urlscan.io phish feed.
+
 ## Examples
 
 ### Aasciinema cast

data/lib/miteru/configuration.rb

@@ -28,6 +28,8 @@ module Miteru
     # @return [Boolean]
     attr_accessor :verbose
 
+    attr_reader :valid_extensions
+
     def initialize
       @auto_download = false
       @ayashige = false
@@ -37,6 +39,8 @@ module Miteru
       @size = 100
       @threads = Parallel.processor_count
       @verbose = false
+
+      @valid_extensions = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
     end
 
     def auto_download?

data/lib/miteru/feeds.rb

@@ -1,8 +1,11 @@
 # frozen_string_literal: true
 
 require_relative "./feeds/feed"
+require_relative "./feeds/phishing_database"
+require_relative "./feeds/phishstats"
 require_relative "./feeds/ayashige"
 require_relative "./feeds/urlscan"
+require_relative "./feeds/urlscan_pro"
 
 module Miteru
   class Feeds
@@ -10,7 +13,10 @@ module Miteru
 
     def initialize
       @feeds = [
+        PhishingDatabase.new,
+        PhishStats.new,
         UrlScan.new(Miteru.configuration.size),
+        UrlScanPro.new,
         Miteru.configuration.ayashige? ? Ayashige.new : nil
       ].compact
     end
@@ -43,6 +49,7 @@ module Miteru
       return [base] if segments.length.zero?
 
       urls = (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join('/')}" }
+
       urls.reject do |breakdowned_url|
         # Reject a url which ends with specific extension names
         invalid_extension? breakdowned_url

data/lib/miteru/feeds/phishing_database.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require "json"
+require "uri"
+
+module Miteru
+  class Feeds
+    class PhishingDatabase < Feed
+      URL = "https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/phishing-links-NEW-today.txt"
+
+      def urls
+        body = get(URL)
+        body.to_s.lines.map(&:chomp)
+      rescue HTTPResponseError, HTTP::Error, JSON::ParserError => e
+        puts "Failed to load phishing database feed (#{e})"
+        []
+      end
+    end
+  end
+end

data/lib/miteru/feeds/phishstats.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "json"
+require "uri"
+
+module Miteru
+  class Feeds
+    class PhishStats < Feed
+      URL = "https://phishstats.info:2096/api/phishing?_sort=-id&size=100"
+
+      def urls
+        json = JSON.parse(get(URL))
+        json.map do |entry|
+          entry.dig("url")
+        end
+      rescue HTTPResponseError, HTTP::Error, JSON::ParserError => e
+        puts "Failed to load PhishStats feed (#{e})"
+        []
+      end
+
+      private
+
+      def url_for(path)
+        URI(URL + path)
+      end
+    end
+  end
+end

data/lib/miteru/feeds/urlscan.rb

@@ -1,39 +1,35 @@
 # frozen_string_literal: true
 
-require "json"
-require "uri"
+require "urlscan"
 
 module Miteru
   class Feeds
     class UrlScan < Feed
-      HOST = "urlscan.io"
-      VERSION = 1
-      URL = "https://#{HOST}/api/v#{VERSION}"
-
       attr_reader :size
+
       def initialize(size = 100)
         @size = size
         raise ArgumentError, "size must be less than 10,000" if size > 10_000
       end
 
+      def api
+        @api ||= ::UrlScan::API.new
+      end
+
       def urls
-        url = url_for("/search/")
-        url.query = URI.encode_www_form(
-          q: "task.method:automatic",
-          size: size
-        )
-
-        res = JSON.parse(get(url))
-        res["results"].map { |result| result.dig("task", "url") }
-      rescue HTTPResponseError, HTTP::Error, JSON::ParserError => e
+        urls_from_community_feed
+      rescue ::UrlScan::ResponseError => e
         puts "Failed to load urlscan.io feed (#{e})"
         []
       end
 
       private
 
-      def url_for(path)
-        URI(URL + path)
+      def urls_from_community_feed
+        res = api.search("task.method:automatic", size: size)
+
+        results = res["results"] || []
+        results.map { |result| result.dig("task", "url") }
       end
     end
   end

data/lib/miteru/feeds/urlscan_pro.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require "urlscan"
+
+module Miteru
+  class Feeds
+    class UrlScanPro < Feed
+      def api
+        @api ||= ::UrlScan::API.new
+      end
+
+      def urls
+        urls_from_pro_feed
+      rescue ::UrlScan::ResponseError => e
+        puts "Failed to load urlscan.io pro feed (#{e})"
+        []
+      end
+
+      private
+
+      def api_key?
+        ENV.key? "URLSCAN_API_KEY"
+      end
+
+      def urls_from_pro_feed
+        return [] unless api_key?
+
+        res = api.pro.phishfeed
+        results = res["results"] || []
+        results.map { |result| result.dig("page_url") }
+      rescue ArgumentError => _e
+        []
+      end
+    end
+  end
+end

data/lib/miteru/http_client.rb

@@ -18,11 +18,20 @@ module Miteru
     end
 
     def download(url, destination)
-      down = Down::Http.new(default_options) { |client| client.headers(default_headers) }
+      down = Down::Http.new(**default_options) { |client| client.headers(**default_headers) }
       down.download(url, destination: destination)
       destination
     end
 
+    def head(url, options = {})
+      options = options.merge default_options
+
+      HTTP.follow
+          .timeout(3)
+          .headers(urlscan_url?(url) ? urlscan_headers : default_headers)
+          .head(url, options)
+    end
+
     def get(url, options = {})
       options = options.merge default_options
 
@@ -48,6 +57,10 @@ module Miteru
       def post(url, options = {})
         new.post url, options
       end
+
+      def head(url, options = {})
+        new.head url, options
+      end
     end
 
     private

data/lib/miteru/kit.rb

@@ -5,37 +5,32 @@ require "securerandom"
 
 module Miteru
   class Kit
-    VALID_EXTENSIONS = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
+    VALID_EXTENSIONS = Miteru.configuration.valid_extensions
 
-    attr_reader :base_url, :link
+    attr_reader :url
 
-    def initialize(base_url:, link:)
-      @base_url = base_url
-      @link = link.start_with?("/") ? link[1..-1] : link
+    def initialize(url)
+      @url = url
     end
 
-    def valid?
-      VALID_EXTENSIONS.include? extname
+    def valid?;
+      valid_ext? && reachable?
     end
 
     def extname
-      return ".tar.gz" if link.end_with?("tar.gz")
+      return ".tar.gz" if url.end_with?("tar.gz")
 
-      File.extname(link)
+      File.extname(url)
     end
 
     def basename
-      File.basename(link)
+      File.basename(url)
     end
 
     def filename
       CGI.unescape basename
     end
 
-    def url
-      "#{base_url}/#{basename}"
-    end
-
     def download_filepath
       "#{base_dir}/#{download_filename}"
     end
@@ -59,7 +54,7 @@ module Miteru
     end
 
     def hostname
-      URI(base_url).hostname
+      URI(url).hostname
     end
 
     def download_filename
@@ -69,5 +64,16 @@ module Miteru
     def base_dir
       @base_dir ||= Miteru.configuration.download_to
     end
+
+    def valid_ext?
+      VALID_EXTENSIONS.include? extname
+    end
+
+    def reachable?
+      res = HTTPClient.head(url)
+      res.status.success?
+    rescue StandardError
+      false
+    end
   end
 end

data/lib/miteru/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miteru
-  VERSION = "0.12.13"
+  VERSION = "0.14.3"
 end

data/lib/miteru/website.rb

@@ -4,7 +4,10 @@ require "oga"
 
 module Miteru
   class Website
+    VALID_EXTENSIONS = Miteru.configuration.valid_extensions
+
     attr_reader :url
+
     def initialize(url)
       @url = url
     end
@@ -15,7 +18,7 @@ module Miteru
 
     def kits
       @kits ||= links.map do |link|
-        kit = Kit.new(base_url: url, link: link.to_s)
+        kit = Kit.new(link)
         kit.valid? ? kit : nil
       end.compact
     end
@@ -33,7 +36,7 @@ module Miteru
     end
 
     def has_kits?
-      ok? && index? && kits?
+      kits?
     rescue Addressable::URI::InvalidURIError, ArgumentError, Encoding::CompatibilityError, HTTP::Error, LL::ParserError, OpenSSL::SSL::SSLError => _e
       false
     end
@@ -46,6 +49,10 @@ module Miteru
       "It might contain #{noun}: #{filename_with_sizes}."
     end
 
+    def links
+      (href_links + possible_file_links).compact.uniq
+    end
+
     private
 
     def response
@@ -66,12 +73,31 @@ module Miteru
       nil
     end
 
-    def links
-      if doc
-        doc.css("a").map { |a| a.get("href") }.compact
+    def href_links
+      if doc && ok? && index?
+        doc.css("a").map { |a| a.get("href") }.compact.map do |href|
+          href = href.start_with?("/") ? href : "/#{href}"
+          url + href
+        end
       else
         []
       end
+    rescue Addressable::URI::InvalidURIError, ArgumentError, Encoding::CompatibilityError, HTTP::Error, LL::ParserError, OpenSSL::SSL::SSLError => _e
+      []
+    end
+
+    def possible_file_links
+      uri = URI.parse(url)
+
+      segments = uri.path.split("/")
+      return [] if segments.length.zero?
+
+      last = segments.last
+      VALID_EXTENSIONS.map do |ext|
+        new_segments = segments[0..-2] + ["#{last}#{ext}"]
+        uri.path = new_segments.join("/")
+        uri.to_s
+      end
     end
   end
 end

data/miteru.gemspec

@@ -24,19 +24,20 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 2.0"
+  spec.add_development_dependency "bundler", "~> 2.1"
   spec.add_development_dependency "coveralls", "~> 0.8"
   spec.add_development_dependency "glint", "~> 0.1"
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.9"
-  spec.add_development_dependency "vcr", "~> 5.0"
-  spec.add_development_dependency "webmock", "~> 3.7"
+  spec.add_development_dependency "vcr", "~> 6.0"
+  spec.add_development_dependency "webmock", "~> 3.8"
 
   spec.add_dependency "colorize", "~> 0.8"
-  spec.add_dependency "down", "~> 5.0"
-  spec.add_dependency "http", "~> 4.2"
-  spec.add_dependency "oga", "~> 2.15"
+  spec.add_dependency "down", "~> 5.1"
+  spec.add_dependency "http", "~> 4.4"
+  spec.add_dependency "oga", "~> 3.2"
   spec.add_dependency "parallel", "~> 1.19"
   spec.add_dependency "slack-notifier", "~> 2.3"
-  spec.add_dependency "thor", "~> 0.20"
+  spec.add_dependency "thor", "~> 1.0"
+  spec.add_dependency "urlscan", "~> 0.5"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miteru
 version: !ruby/object:Gem::Version
-  version: 0.12.13
+  version: 0.14.3
 platform: ruby
 authors:
 - Manabu Niseki
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2019-11-27 00:00:00.000000000 Z
+date: 2020-06-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '2.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '2.1'
 - !ruby/object:Gem::Dependency
   name: coveralls
   requirement: !ruby/object:Gem::Requirement
@@ -86,28 +86,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '6.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '6.0'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.7'
+        version: '3.8'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.7'
+        version: '3.8'
 - !ruby/object:Gem::Dependency
   name: colorize
   requirement: !ruby/object:Gem::Requirement
@@ -128,42 +128,42 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '5.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '5.1'
 - !ruby/object:Gem::Dependency
   name: http
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.2'
+        version: '4.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.2'
+        version: '4.4'
 - !ruby/object:Gem::Dependency
   name: oga
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.15'
+        version: '3.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.15'
+        version: '3.2'
 - !ruby/object:Gem::Dependency
   name: parallel
   requirement: !ruby/object:Gem::Requirement
@@ -198,14 +198,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.20'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.20'
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: urlscan
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
 description: An experimental phishing kit detector
 email:
 - manabu.niseki@gmail.com
@@ -235,7 +249,10 @@ files:
 - lib/miteru/feeds.rb
 - lib/miteru/feeds/ayashige.rb
 - lib/miteru/feeds/feed.rb
+- lib/miteru/feeds/phishing_database.rb
+- lib/miteru/feeds/phishstats.rb
 - lib/miteru/feeds/urlscan.rb
+- lib/miteru/feeds/urlscan_pro.rb
 - lib/miteru/http_client.rb
 - lib/miteru/kit.rb
 - lib/miteru/notifier.rb
@@ -247,7 +264,7 @@ homepage: https://github.com/ninoseki/miteru
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -262,8 +279,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.1.2
+signing_key:
 specification_version: 4
 summary: An experimental phishing kit detector
 test_files: []