miteru 0.9.6 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cb9c20111a24e67de7f13cfec1a28f59dd45760bf8f49ebb73a04e1dd8fc6cd3
-  data.tar.gz: '0801d93ed7f76d276daa3e6d19f122448d2be6bcab8a030f624b5b841dc91156'
+  metadata.gz: f3388ff4a8c19447535952bc611725e1faa31b04241b0432cb7516ed03222f1f
+  data.tar.gz: 28440e7ff65d33a303e5446b4c9eb454ee0831b6b2acc4bef938b855a9b08d23
 SHA512:
-  metadata.gz: 28e0951fb925046647f5ecc45da42c4777106925890ec644925d3f82f8b7c6b44dc47c41075b589d08fa7a92de8c0a1cba4481cda92e0094b355c2e60f9e831a
-  data.tar.gz: 1ede811bcb66b49193be59218d0313d486b296d54de6b6861fb6e1fd2b8778326308bd7817abb392d3ded522aacb4605178f0fbd308de22b6f0841b975027b4f
+  metadata.gz: dc88d56a0b8d3d22ea703e8a4d60671804cda0d2ca8323733580260815aad0a0046757c31a35872140c1bee624b2ff17894eecd37c25390f3b85634bd1c2d0c3
+  data.tar.gz: ee0de0987aea58886a61d2c95e4df80a81d4cf8941342b1c0e48ef8528f21db411c59e047a423f6dbecfdeee79d98b4ed5a6587c6a741e8147d6b440c2b44752

data/README.md

@@ -11,8 +11,8 @@ Miteru is an experimental phishing kit detection tool.
 
 - It collects phishing suspicious URLs from the following feeds:
   - [urlscan.io certstream-suspicious feed](https://urlscan.io/search/#certstream-suspicious)
-  - [OpenPhish community feed](https://openphish.com/feed.txt)
-  - [PhishTank feed](http://data.phishtank.com/data/online-valid.csv)
+  - [OpenPhish feed via urlscan.io](https://urlscan.io/search/#OpenPhish)
+  - [PhishTank feed via urlscan.io](https://urlscan.io/search/#PhishTank)
 - It checks a suspicious URL whether it enables directory listing and contains a phishing kit (compressed file) or not.
   - Note: compressed file = `*.zip`, `*.rar`, `*.7z`, `*.tar` and `*.gz`.
 

data/lib/miteru/downloader.rb

@@ -15,13 +15,15 @@ module Miteru
     def download_compressed_files(url, compressed_files)
       compressed_files.each do |path|
         target_url = "#{url}/#{path}"
+        filename = filename_to_save(target_url)
+        destination = filepath_to_download(filename)
         begin
-          download_file_path = HTTPClient.download(target_url, base_dir)
-          if duplicated?(download_file_path, base_dir)
-            puts "Do not download #{target_url} because there is a same hash file in the directory (SHA256: #{sha256(download_file_path)})."
-            FileUtils.rm download_file_path
+          download_filepath = HTTPClient.download(target_url, destination)
+          if duplicated?(download_filepath)
+            puts "Do not download #{target_url} because there is a same hash file in the directory (SHA256: #{sha256(download_filepath)})."
+            FileUtils.rm download_filepath
           else
-            puts "Download #{target_url} as #{download_file_path}"
+            puts "Download #{target_url} as #{download_filepath}"
           end
         rescue Down::Error => e
           puts "Failed to download: #{target_url} (#{e})"
@@ -31,12 +33,22 @@ module Miteru
 
     private
 
+    def filename_to_save(url)
+      filename = url.split("/").last
+      extname = File.extname(filename)
+      "#{SecureRandom.alphanumeric}#{extname}"
+    end
+
+    def filepath_to_download(filename)
+      "#{base_dir}/#{filename}"
+    end
+
     def sha256(path)
       digest = Digest::SHA256.file(path)
       digest.hexdigest
     end
 
-    def duplicated?(file_path, base_dir)
+    def duplicated?(file_path)
       base = sha256(file_path)
       sha256s = Dir.glob("#{base_dir}/*.{zip,rar,7z,tar,gz}").map { |path| sha256(path) }
       sha256s.select { |sha256| sha256 == base }.length > 1

data/lib/miteru/feeds.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
 require_relative "./feeds/feed"
-require_relative "./feeds/openphish"
-require_relative "./feeds/phishtank"
 require_relative "./feeds/urlscan"
 
 module Miteru
@@ -11,15 +9,13 @@ module Miteru
     attr_reader :directory_traveling
 
     def initialize(urlscan_size = 100, directory_traveling: false)
-      @openphish = OpenPhish.new
-      @phishtank = PhishTank.new
       @urlscan = UrlScan.new(urlscan_size)
       @directory_traveling = directory_traveling
     end
 
     def suspicious_urls
       @suspicious_urls ||= [].tap do |arr|
-        urls = (openphish.urls + phishtank.urls + urlscan.urls).select { |url| url.start_with?("http://", "https://") }
+        urls = urlscan.urls.select { |url| url.start_with?("http://", "https://") }
         urls.map { |url| breakdown(url) }.flatten.uniq.sort.each { |url| arr << url }
       end
     end

data/lib/miteru/feeds/urlscan.rb

@@ -1,11 +1,14 @@
 # frozen_string_literal: true
 
 require "json"
+require "uri"
 
 module Miteru
   class Feeds
     class UrlScan < Feed
-      ENDPOINT = "https://urlscan.io/api/v1"
+      HOST = "urlscan.io"
+      VERSION = 1
+      URL = "https://#{HOST}/api/v#{VERSION}"
 
       attr_reader :size
       def initialize(size = 100)
@@ -14,13 +17,24 @@ module Miteru
       end
 
       def urls
-        url = "#{ENDPOINT}/search/?q=certstream-suspicious&size=#{size}"
+        url = url_for("/search/")
+        url.query = URI.encode_www_form(
+          q: "PhishTank OR OpenPhish OR CertStream-Suspicious",
+          size: size
+        )
+
         res = JSON.parse(get(url))
         res["results"].map { |result| result.dig("task", "url") }
       rescue HTTPResponseError => e
         puts "Failed to load urlscan.io feed (#{e})"
         []
       end
+
+      private
+
+      def url_for(path)
+        URI(URL + path)
+      end
     end
   end
 end

data/lib/miteru/http_client.rb

@@ -14,8 +14,7 @@ module Miteru
       @ssl_context = ctx
     end
 
-    def download(url, base_dir)
-      destination = download_path(base_dir, filename_to_save(url))
+    def download(url, destination)
       down = Down::Http.new(default_options) { |client| client.headers(default_headers) }
       down.download(url, destination: destination)
       destination
@@ -51,15 +50,5 @@ module Miteru
     def default_options
       { ssl_context: ssl_context }
     end
-
-    def filename_to_save(url)
-      filename = url.split("/").last
-      extname = File.extname(filename)
-      "#{SecureRandom.alphanumeric}#{extname}"
-    end
-
-    def download_path(base_dir, filename)
-      "#{base_dir}/#{filename}"
-    end
   end
 end

data/lib/miteru/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miteru
-  VERSION = "0.9.6"
+  VERSION = "0.10.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miteru
 version: !ruby/object:Gem::Version
-  version: 0.9.6
+  version: 0.10.0
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-11-13 00:00:00.000000000 Z
+date: 2018-11-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -217,8 +217,6 @@ files:
 - lib/miteru/error.rb
 - lib/miteru/feeds.rb
 - lib/miteru/feeds/feed.rb
-- lib/miteru/feeds/openphish.rb
-- lib/miteru/feeds/phishtank.rb
 - lib/miteru/feeds/urlscan.rb
 - lib/miteru/http_client.rb
 - lib/miteru/version.rb

data/lib/miteru/feeds/openphish.rb

@@ -1,19 +0,0 @@
-# frozen_string_literal: true
-
-require "json"
-
-module Miteru
-  class Feeds
-    class OpenPhish < Feed
-      ENDPOINT = "https://openphish.com"
-
-      def urls
-        res = get("#{ENDPOINT}/feed.txt")
-        res.lines.map(&:chomp)
-      rescue HTTPResponseError => e
-        puts "Failed to load OpenPhish feed (#{e})"
-        []
-      end
-    end
-  end
-end

data/lib/miteru/feeds/phishtank.rb

@@ -1,20 +0,0 @@
-# frozen_string_literal: true
-
-require "csv"
-
-module Miteru
-  class Feeds
-    class PhishTank < Feed
-      ENDPOINT = "http://data.phishtank.com"
-
-      def urls
-        res = get("#{ENDPOINT}/data/online-valid.csv")
-        table = CSV.parse(res, headers: true)
-        table.map { |row| row["url"] }
-      rescue HTTPResponseError => e
-        puts "Failed to load PhishTank feed (#{e})"
-        []
-      end
-    end
-  end
-end