miteru 0.9.3 → 0.9.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 89ed6dea77f4809ef7cacfd7543a35d96f8032807d6cc93fe5e6aa52e752d5d1
-  data.tar.gz: 812d51bdbc1c245c87f7a3fcc7e70577536cc8eacd0cfc9d10b6be3b146b96d9
+  metadata.gz: e11aca7b7b5b2222d5733afd3b155420475635d3b4f09b02a70e3017fb9e68cf
+  data.tar.gz: 0e0d8b5b4acb5033f1e2639563e6dbd5ec2f6dfabb4421315f18ed45648076f8
 SHA512:
-  metadata.gz: cb94419c132e650a1e57f3bccd134b91be5847ab9197ceb69c71a08b8bfec3667a5e63f504892ef4d08a74f049f4bc979de4ac7c145a2a30502ed92b91f73c78
-  data.tar.gz: 60b012761f7eced3acf58a7acf4487b2d0466a9ebc2fa1b9fe9b23e3b315b3ed4d5bb798e1786788c904691656c989f7ddda962a52ca307abe4e762d7281ad65
+  metadata.gz: '08d110df3c96cc33c0e9fefefb95f1dff64c8482c423ec8a725497add862d648464c6a5a164371e0f664b38c83fcf0becc8d3ee53f7d233f9776e3ab1aea12a0'
+  data.tar.gz: 2e45e1ae4d80c032abeef6287d705f970847f8f6bc237f82ea7749492cf4fc7e07eba8e37daf1275d5656a04212deb5cba2e5de5ca75b2c4e5ad49bb72d2efba

data/README.md

@@ -10,12 +10,18 @@ Miteru is an experimental phishing kit detection tool.
 ## How it works
 
 - It collects phishing suspicious URLs from the following feeds:
-  - [urlscan.io](https://urlscan.io/search/#certstream-suspicious)
+  - [urlscan.io certstream-suspicious feed](https://urlscan.io/search/#certstream-suspicious)
   - [OpenPhish community feed](https://openphish.com/feed.txt)
   - [PhishTank feed](http://data.phishtank.com/data/online-valid.csv)
-- It checks a suspicious URL whether it has a directory listing and contains a phishing kit (compressed file) or not.
+- It checks a suspicious URL whether it enables directory listing and contains a phishing kit (compressed file) or not.
   - Note: compressed file = `*.zip`, `*.rar`, `*.7z`, `*.tar` and `*.gz`.
 
+## Features
+
+- [x] Phishing kit detection & collection.
+- [x] Slack integration. (Posting a message to Slack if the tool detects a phishing kit.)
+- [x] Threading.
+
 ## Installation
 
 ```sh
@@ -61,9 +67,20 @@ https://dummy3.com: it doesn't contain a phishing kit.
 https://dummy4.com: it might contain a phishing kit (dummy.zip).
 ```
 
+## Aasciinema cast
+
+[![asciicast](https://asciinema.org/a/ga6ZbwuK1HOLOyELb23QrSvJP.svg)](https://asciinema.org/a/ga6ZbwuK1HOLOyELb23QrSvJP)
+
+Note: Stoped the process during the execution because it takes minutes to finish.
+
 ## Note
 
 For using `--post-to-slack` feature, you should set the following environment variables:
 
 - `SLACK_WEBHOOK_URL`: Your Slack Webhook URL.
 - `SLACK_CHANNEL`: Slack channel to post a message (default: "#general").
+
+## Alternatives
+
+- [t4d/StalkPhish](https://github.com/t4d/StalkPhish): The Phishing kits stalker, harvesting phishing kits for investigations.
+- [duo-labs/phish-collect](https://github.com/duo-labs/phish-collect): Python script to hunt phishing kits.

data/lib/miteru/crawler.rb

@@ -75,7 +75,5 @@ module Miteru
     def valid_slack_setting?
       ENV["SLACK_WEBHOOK_URL"] != nil
     end
-
-
   end
 end

data/lib/miteru/downloader.rb

@@ -17,6 +17,7 @@ module Miteru
         target_url = "#{url}/#{path}"
         begin
           download_file_path = HTTPClient.download(target_url, base_dir)
+          puts download_file_path
           if duplicated?(download_file_path, base_dir)
             puts "Do not download #{target_url} because there is a same hash file in the directory (SHA256: #{sha256(download_file_path)})."
             FileUtils.rm download_file_path
@@ -38,7 +39,7 @@ module Miteru
 
     def duplicated?(file_path, base_dir)
       base = sha256(file_path)
-      sha256s = Dir.glob("#{base_dir}/*.zip").map { |path| sha256(path) }
+      sha256s = Dir.glob("#{base_dir}/*.{zip,rar,7z,tar,gz}").map { |path| sha256(path) }
       sha256s.select { |sha256| sha256 == base }.length > 1
     end
   end

data/lib/miteru/http_client.rb

@@ -15,7 +15,7 @@ module Miteru
     end
 
     def download(url, base_dir)
-      destination = download_to(base_dir, save_filename)
+      destination = download_path(base_dir, filename_to_save(url))
       down = Down::Http.new(default_options) { |client| client.headers(default_headers) }
       down.download(url, destination: destination)
       destination
@@ -52,12 +52,14 @@ module Miteru
       { ssl_context: ssl_context }
     end
 
-    def save_filename
-      "#{SecureRandom.alphanumeric}.zip"
+    def filename_to_save(url)
+      filename = url.split("/").last
+      extname = File.extname(filename)
+      "#{SecureRandom.alphanumeric}.#{extname}"
     end
 
-    def download_to(base_dir, save_filename)
-      "#{base_dir}/#{save_filename}"
+    def download_path(base_dir, filename)
+      "#{base_dir}/#{filename}"
     end
   end
 end

data/lib/miteru/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miteru
-  VERSION = "0.9.3"
+  VERSION = "0.9.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miteru
 version: !ruby/object:Gem::Version
-  version: 0.9.3
+  version: 0.9.4
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-10-27 00:00:00.000000000 Z
+date: 2018-11-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler