miteru 0.6.1 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cc8a49cc98035612ecb5f3a085b885175ca8c69d426f6a9b059a4db7b79ab9fa
-  data.tar.gz: 268d4db3a53946e56fdaa796a6664928fe49ef8a80c9337bd7cdc0221fb0c684
+  metadata.gz: aab5b29928c45f80d099e6c12dee3606fd3002696814c3ea1d937221a28dc2d0
+  data.tar.gz: eba0390423b107ac8d1df75bc39f47db8b182070aeadee56396cc24f174d6e70
 SHA512:
-  metadata.gz: c9a7685eb8b46fb553472a10d3fa4efe8252be5aef403f69ad7e92d37225352df528070039763f53389182405f5e19df72a24ffb17e63bf336c3af9d378e244c
-  data.tar.gz: e7e4959a8d6db5ca19b8fa670117b3e1206d8ebbe0ec185c5c18db0be74175a07f0dfb9207b8c631d43f67037099b5e12151ba2f91df9a35ef8dfbdc110342e6
+  metadata.gz: 00f9affdcae054c0570965f60afbfcf3c25fcdf57ad4a26b7d9411a54d917593b37bc861f3f3ef1d467817c00e18c88648b52cd15bd0fac087213adb722a3cb4
+  data.tar.gz: 49a9618ee2aa8f7914cc8341b04f8803c965ce5e2cb3f53469f27568e73ca2ccdc9a26a3b70a7bc576c2fe95049f7837bccdc74352eb6d4529e57fe7e2da70da

data/README.md

@@ -33,16 +33,17 @@ Usage:
   miteru execute
 
 Options:
-  [--auto-download], [--no-auto-download]  # Enable or disable auto-download of *.zip file(s)
-  [--download-to=DOWNLOAD_TO]              # Directory to download file(s)
-                                           # Default: /tmp
-  [--post-to-slack], [--no-post-to-slack]  # Post a message to Slack if it detects a phishing kit
-  [--size=N]                               # Number of urlscan.io's results. (Max: 100,000)
-                                           # Default: 100
-  [--threads=N]                            # Number of threads to use
-                                           # Default: 10
+  [--auto-download], [--no-auto-download]              # Enable or disable auto-download of *.zip file(s)
+  [--directory-traveling], [--no-directory-traveling]  # Enable or disable directory traveling
+  [--download-to=DOWNLOAD_TO]                          # Directory to download file(s)
+                                                       # Default: /tmp
+  [--post-to-slack], [--no-post-to-slack]              # Post a message to Slack if it detects a phishing kit
+  [--size=N]                                           # Number of urlscan.io's results. (Max: 100,000)
+                                                       # Default: 100
+  [--threads=N]                                        # Number of threads to use
+                                                       # Default: 10
   [--verbose], [--no-verbose]
-                                           # Default: true
+                                                       # Default: true
 
 Execute the crawler
 ```

data/lib/miteru/cli.rb

@@ -9,6 +9,7 @@ require "thor"
 module Miteru
   class CLI < Thor
     method_option :auto_download, type: :boolean, default: false, desc: "Enable or disable auto-download of *.zip file(s)"
+    method_option :directory_traveling, type: :boolean, default: false, desc: "Enable or disable directory traveling"
     method_option :download_to, type: :string, default: "/tmp", desc: "Directory to download file(s)"
     method_option :post_to_slack, type: :boolean, default: false, desc: "Post a message to Slack if it detects a phishing kit"
     method_option :size, type: :numeric, default: 100, desc: "Number of urlscan.io's results. (Max: 100,000)"
@@ -16,7 +17,12 @@ module Miteru
     method_option :verbose, type: :boolean, default: true
     desc "execute", "Execute the crawler"
     def execute
-      websites = Crawler.execute(size: options[:size], threads: options[:threads], verbose: options[:verbose])
+      websites = Crawler.execute(
+        directory_traveling: options[:directory_traveling],
+        size: options[:size],
+        threads: options[:threads],
+        verbose: options[:verbose]
+      )
       websites.each do |website|
         next unless website.has_kit?
 

data/lib/miteru/crawler.rb

@@ -7,14 +7,16 @@ require "uri"
 
 module Miteru
   class Crawler
-    attr_reader :threads
+    attr_reader :directory_traveling
     attr_reader :size
+    attr_reader :threads
     attr_reader :verbose
 
     URLSCAN_ENDPOINT = "https://urlscan.io/api/v1"
     OPENPHISH_ENDPOINT = "https://openphish.com"
 
-    def initialize(size: 100, threads: 10, verbose: false)
+    def initialize(directory_traveling: false, size: 100, threads: 10, verbose: false)
+      @directory_traveling = directory_traveling
       @size = size
       @threads = threads
       @verbose = verbose
@@ -39,14 +41,16 @@ module Miteru
         return []
       end
       base = "#{uri.scheme}://#{uri.hostname}"
-      [base]
-      # TODO: Should add a option for burute force directory
-      # segments = uri.path.split("/")
-      # if segments.length.zero?
-      #   [base]
-      # else
-      #   (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join('/')}" }
-      # end
+      return [base] unless directory_traveling
+
+      segments = uri.path.split("/")
+      return [base] if segments.length.zero?
+
+      urls = (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join('/')}" }
+      urls.reject do |breakdowned_url|
+        # Reject a url which ends with specific extension names
+        %w(.htm .html .php .asp .aspx).any? { |ext| breakdowned_url.end_with? ext }
+      end
     end
 
     def suspicious_urls
@@ -73,8 +77,8 @@ module Miteru
       websites
     end
 
-    def self.execute(size: 100, threads: 10, verbose: false)
-      new(size: size, threads: threads, verbose: verbose).execute
+    def self.execute(directory_traveling: false, size: 100, threads: 10, verbose: false)
+      new(directory_traveling: directory_traveling, size: size, threads: threads, verbose: verbose).execute
     end
 
     private

data/lib/miteru/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miteru
-  VERSION = "0.6.1"
+  VERSION = "0.7.0"
 end

data/lib/miteru/website.rb

@@ -36,7 +36,7 @@ module Miteru
     end
 
     def has_kit?
-      ok? && index? && zip_files?
+      @has_kit ||= ok? && index? && zip_files?
     end
 
     def build

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miteru
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 0.7.0
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-09-25 00:00:00.000000000 Z
+date: 2018-09-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler