miteru 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aab5b29928c45f80d099e6c12dee3606fd3002696814c3ea1d937221a28dc2d0
-  data.tar.gz: eba0390423b107ac8d1df75bc39f47db8b182070aeadee56396cc24f174d6e70
+  metadata.gz: efab1503dd6c96a4376d6baed09eadccde40d9a43440604e45c2926bdb301209
+  data.tar.gz: fda7418c3101bd3e691bd6cb0af7b3b2a6b3c0cd58da30d7a2171868251468c4
 SHA512:
-  metadata.gz: 00f9affdcae054c0570965f60afbfcf3c25fcdf57ad4a26b7d9411a54d917593b37bc861f3f3ef1d467817c00e18c88648b52cd15bd0fac087213adb722a3cb4
-  data.tar.gz: 49a9618ee2aa8f7914cc8341b04f8803c965ce5e2cb3f53469f27568e73ca2ccdc9a26a3b70a7bc576c2fe95049f7837bccdc74352eb6d4529e57fe7e2da70da
+  metadata.gz: 0b10ef50e80806ca90386c4da6ae9057e044af58a1975d3c8b4c03942966b09e23386a64d233916610066ded7c7ec59700ddb4b149107cee71acb6a52459d139
+  data.tar.gz: a977b070175607fed8a2ec30a331278a6f0cd1f6438c1a3b4accffd2be18f8963f92afac01d0144f0567aabefab19c3ed4af4dd4f83c338a3af9bb22022ea542

data/README.md

@@ -9,7 +9,10 @@ Miteru is an experimental phishing kit detection tool.
 
 ## How it works
 
-- It collects phishing suspicious URLs from [urlscan.io](https://urlscan.io/search/#certstream-suspicious) and [OpenPhish community feed](https://openphish.com/feed.txt).
+- It collects phishing suspicious URLs from the following feeds:
+  - [urlscan.io](https://urlscan.io/search/#certstream-suspicious)
+  - [OpenPhish community feed](https://openphish.com/feed.txt)
+  - [PhishTank feed](http://data.phishtank.com/data/online-valid.csv)
 - It checks a suspicious URL whether it has a directory listing and contains a phishing kit (`*.zip` file) or not.
 
 ## Installation

data/lib/miteru/crawler.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "csv"
 require "http"
 require "json"
 require "thread/pool"
@@ -14,6 +15,7 @@ module Miteru
 
     URLSCAN_ENDPOINT = "https://urlscan.io/api/v1"
     OPENPHISH_ENDPOINT = "https://openphish.com"
+    PHISHTANK_ENDPOINT = "http://data.phishtank.com"
 
     def initialize(directory_traveling: false, size: 100, threads: 10, verbose: false)
       @directory_traveling = directory_traveling
@@ -34,6 +36,12 @@ module Miteru
       res.lines.map(&:chomp)
     end
 
+    def phishtank_feed
+      res = get("#{PHISHTANK_ENDPOINT}/data/online-valid.csv")
+      table = CSV.parse(res, headers: true)
+      table.map { |row| row["url"] }
+    end
+
     def breakdown(url)
       begin
         uri = URI.parse(url)
@@ -54,7 +62,7 @@ module Miteru
     end
 
     def suspicious_urls
-      urls = urlscan_feed + openphish_feed
+      urls = (urlscan_feed + openphish_feed + phishtank_feed)
       urls.map { |url| breakdown(url) }.flatten.uniq.sort
     end
 
@@ -65,11 +73,12 @@ module Miteru
       suspicious_urls.each do |url|
         pool.process do
           website = Website.new(url)
-          unless website.has_kit?
+          if website.has_kit?
+            websites << website
+          else
             puts "#{website.url}: it doesn't contain a phishing kit." if verbose
             website.unbuild
           end
-          websites << website
         end
       end
       pool.shutdown
@@ -84,7 +93,7 @@ module Miteru
     private
 
     def get(url)
-      res = HTTP.get(url)
+      res = HTTP.follow(max_hops: 3).get(url)
       raise HTTPResponseError if res.code != 200
 
       res.body.to_s

data/lib/miteru/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miteru
-  VERSION = "0.7.0"
+  VERSION = "0.8.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miteru
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-09-28 00:00:00.000000000 Z
+date: 2018-10-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler