yawast 0.4.0.beta2 → 0.4.0.beta3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0bd401e7dfb0ed1202b3649590c6188004f32d89
-  data.tar.gz: 349a07e700f68ffaf4cb2c3b2d73723b838eaa63
+  metadata.gz: c657b676e6fb2fab6ddf48ac8c7b9a8a8606e1b7
+  data.tar.gz: 24071878a0ea703638f5ce1587524505489a9cfb
 SHA512:
-  metadata.gz: 08980226efef51d1c4bd02b943fd63ff09b759bc4b921e544e2684f695682867214970d14e7ea31e22a97ce79720db1cb03de3a143fefa5ed0dc7652d8d1e96b
-  data.tar.gz: 77de689186f88cbb65b6ec5dc072c50013696b4f50420cd982afd0b6eb1c2be888cf7e99c09bc8dd8d8c7978c9c9014b471f19a5a214acf6f8be190822ed103b
+  metadata.gz: df6f61d22b221f9dccaff5ef22fdc0708793034368d1101762fdb41338c8a5472fd1ba81cf2ec746a3c76521e5a9ae288e4b39c5196426ca66d59d580eb1cb70
+  data.tar.gz: 571fff42d3a0a67c5c1c8ec7a1d30994e91a73ba1a186f150b120a7f3edb83149a591ca631377cb69f49b1e590dd5de30a19bc2c9fc0e7d021b847e59a33c9a9

data/CHANGELOG.md

@@ -1,5 +1,7 @@
 ## 0.4.0 - In Development
 
+* [#66](https://github.com/adamcaudill/yawast/issues/66) - Thread directory search for better performance
+* [#67](https://github.com/adamcaudill/yawast/issues/67) - Make "Found Redirect" optional
 * [#65](https://github.com/adamcaudill/yawast/issues/65) - Bug: Output redirection doesn't work correctly
 
 ## 0.3.0 - 2016-09-15

data/README.md

@@ -207,7 +207,7 @@ This mode is the most comprehensive, and contains far more data than the Interna
 
 ### Usage
 
-* Standard scan: `./yawast scan <url> [--internalssl] [--tdessessioncount] [--nossl] [--nociphers] [--dir] [--proxy localhost:8080] [--cookie SESSIONID=12345]`
+* Standard scan: `./yawast scan <url> [--internalssl] [--tdessessioncount] [--nossl] [--nociphers] [--dir] [--dirrecursive] [--dirlistredir] [--proxy localhost:8080] [--cookie SESSIONID=12345]`
 * HEAD-only scan: `./yawast head <url> [--internalssl] [--tdessessioncount] [--nossl] [--nociphers] [--proxy localhost:8080] [--cookie SESSIONID=12345]`
 * SSL information: `./yawast ssl <url> [--internalssl] [--tdessessioncount] [--nociphers]`
 * CMS detection: `./yawast cms <url> [--proxy localhost:8080] [--cookie SESSIONID=12345]`

data/bin/yawast

@@ -19,6 +19,7 @@ command :scan do |c|
   c.option '--tdessessioncount', 'Counts the number of messages that can be sent in a single session'
   c.option '--dir', 'Enables directory search'
   c.option '--dirrecursive', 'Recursive directory search (only with --dir)'
+  c.option '--dirlistredir', 'Show 301 redirects (only with --dir)'
   c.option '--proxy STRING', String, 'HTTP Proxy Server (such as Burp Suite)'
   c.option '--cookie STRING', String, 'Session cookie'
 

data/lib/scanner/cert.rb

@@ -21,7 +21,7 @@ module Yawast
 
         content = File.readlines options.input
 
-        pool_size = 16
+        pool_size = 32
         jobs = Queue.new
         @results = Queue.new
 
@@ -66,7 +66,7 @@ module Yawast
         return if domain == ''
 
         begin
-          socket = Socket.tcp(domain, 443, opts={connect_timeout: 3})
+          socket = Socket.tcp(domain, 443, opts={connect_timeout: 8})
 
           ctx = OpenSSL::SSL::SSLContext.new
           ctx.ciphers = OpenSSL::SSL::SSLContext::DEFAULT_PARAMS[:ciphers]
@@ -74,7 +74,7 @@ module Yawast
           ssl = OpenSSL::SSL::SSLSocket.new(socket, ctx)
           ssl.hostname = domain
 
-          Timeout::timeout(5) {
+          Timeout::timeout(16) {
             ssl.connect
           }
 

data/lib/scanner/core.rb

@@ -63,7 +63,7 @@ module Yawast
 
             #check for common directories
             if options.dir
-              Yawast::Scanner::Generic.directory_search(@uri, options.dirrecursive)
+              Yawast::Scanner::Plugins::Http::DirectorySearch.search @uri, options.dirrecursive, options.dirlistredir
             end
 
             get_cms(@uri, options)

data/lib/scanner/generic.rb

@@ -212,45 +212,6 @@ module Yawast
         end
       end
 
-      def self.directory_search(uri, recursive, banner = true)
-        if banner
-          if recursive
-            puts 'Recursively searching for common directories (this will take a while)...'
-          else
-            puts 'Searching for common directories...'
-          end
-        end
-
-        begin
-          req = Yawast::Shared::Http.get_http(uri)
-          req.use_ssl = uri.scheme == 'https'
-          req.keep_alive_timeout = 600
-          headers = Yawast::Shared::Http.get_headers
-
-          req.start do |http|
-            File.open(File.dirname(__FILE__) + '/../resources/common.txt', "r") do |f|
-              f.each_line do |line|
-                check = uri.copy
-                check.path = check.path + "#{line.strip}/"
-
-                res = http.head(check, headers)
-
-                if res.code == '200'
-                  Yawast::Utilities.puts_info "\tFound: '#{check.to_s}'"
-                  directory_search check, recursive, false if recursive
-                elsif res.code == '301'
-                  Yawast::Utilities.puts_info "\tFound Redirect: '#{check.to_s} -> '#{res['Location']}'"
-                end
-              end
-            end
-          end
-        rescue => e
-          Yawast::Utilities.puts_error "Error searching for directories (#{e.message})"
-        end
-
-        puts '' if banner
-      end
-
       def self.check_options(uri)
         begin
           req = Yawast::Shared::Http.get_http(uri)

data/lib/scanner/plugins/http/directory_search.rb

@@ -0,0 +1,91 @@
+module Yawast
+  module Scanner
+    module Plugins
+      module Http
+        class DirectorySearch
+          def self.search(uri, recursive, list_redirects)
+            @recursive = recursive
+            @list_redirects = list_redirects
+
+            if recursive
+              puts 'Recursively searching for common directories (this will take a while)...'
+            else
+              puts 'Searching for common directories...'
+            end
+
+            begin
+              pool_size = 16
+              @jobs = Queue.new
+              @results = Queue.new
+
+              #load the queue, starting at /
+              base = uri.copy
+              base.path = '/'
+              load_queue base
+
+              workers = (pool_size).times.map do
+                Thread.new do
+                  begin
+                    while (check = @jobs.pop(true))
+                      process check
+                    end
+                  rescue ThreadError
+                    #do nothing
+                  end
+                end
+              end
+
+              results = Thread.new do
+                begin
+                  while true
+                    if @results.length > 0
+                      out = @results.pop(true)
+                      Yawast::Utilities.puts_info out
+                    end
+                  end
+                rescue ThreadError
+                  #do nothing
+                end
+              end
+
+              workers.map(&:join)
+              results.terminate
+            rescue => e
+              Yawast::Utilities.puts_error "Error searching for directories (#{e.message})"
+            end
+
+            puts
+          end
+
+          def self.load_queue(uri)
+            File.open(File.dirname(__FILE__) + '/../../../resources/common.txt', "r") do |f|
+              f.each_line do |line|
+                check = uri.copy
+                check.path = check.path + "#{line.strip}/"
+
+                #add the job to the queue
+                @jobs.push check
+              end
+            end
+          end
+
+          def self.process(uri)
+            begin
+              res = Yawast::Shared::Http.head uri
+
+              if res.code == '200'
+                @results.push "\tFound: '#{uri.to_s}'"
+
+                load_queue uri if @recursive
+              elsif res.code == '301' && @list_redirects
+                @results.push "\tFound Redirect: '#{uri.to_s} -> '#{res['Location']}'"
+              end
+            rescue => e
+              Yawast::Utilities.puts_error "Error searching for directories (#{e.message})"
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/version.rb

@@ -1,3 +1,3 @@
 module Yawast
-  VERSION = '0.4.0.beta2'
+  VERSION = '0.4.0.beta3'
 end

data/lib/yawast.rb

@@ -2,7 +2,7 @@
 #
 # path - The String relative path from here to the directory.
 def require_all(path)
-  glob = File.join(File.dirname(__FILE__), path, '*.rb')
+  glob = File.join(File.dirname(__FILE__), path + '/**/', '*.rb')
   Dir[glob].each do |f|
     require f
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: yawast
 version: !ruby/object:Gem::Version
-  version: 0.4.0.beta2
+  version: 0.4.0.beta3
 platform: ruby
 authors:
 - Adam Caudill
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-09-21 00:00:00.000000000 Z
+date: 2016-09-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ssllabs
@@ -126,6 +126,7 @@ files:
 - lib/scanner/nginx.rb
 - lib/scanner/obj_presence.rb
 - lib/scanner/php.rb
+- lib/scanner/plugins/http/directory_search.rb
 - lib/scanner/ssl.rb
 - lib/scanner/ssl_labs.rb
 - lib/shared/http.rb