ronin-recon 0.1.0.rc1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1a9c5f1341f2119335b680ceb29bed06bafa19a74664bacf71d4385ec2b517cb
-  data.tar.gz: b0ba0ddde38138cb220ba5dc8b6a31005811622086b37ed51a1de5a3f80cc717
+  metadata.gz: fb12d7c803f46db1479703050ae863e2c684bcf02c27e56845f57aabadba8295
+  data.tar.gz: 3ecef315a0ef47934ba2b6b7117ad9b2bc9813508f4db64d96e851b09469008b
 SHA512:
-  metadata.gz: 9d6dac836e0943da2c2dc1478ce4bc557328811c4a9c4edb035462aaae0cfe3827f8b40acb7863a0ea61060c7f110a934e3f58ed5658decc776595064be664ba
-  data.tar.gz: 13e2d6e10d7c38e70af52a616cc64f07b794d29535d80195d25e33c2e7f04430ef35964c557bcbfc65ac29177fde916141dc9f25fab261535ea590fc38498cd3
+  metadata.gz: 8d4956f0572724eb5db72c774ac20202dfef0d56513d1569a3d309c5329a988a3f085db9c3d58bb23f4f7922a6da87c67f69d43971efff50704e89a2486393e9
+  data.tar.gz: 87eb036e8d7702d7a9bd2645c1bf152b797e990a6bf0a7f566db8c1f787217a9c14b19f9d93940f6f8645745367bba63cf27ee6b9700214d73567a17b324da6f

data/.ruby-version

@@ -1 +1 @@
-ruby-3.1
+ruby-3.3

data/ChangeLog.md

@@ -1,4 +1,4 @@
-### 0.1.0 / 2024-XX-XX
+### 0.1.0 / 2024-07-22
 
 * Initial release:
   * Uses asynchronous I/O and fibers.

data/README.md

@@ -56,6 +56,7 @@ and uses asynchronous I/O to maximize efficiency.
 
 * Does not require API keys to run.
 * Not just a script that runs a bunch of other recon tools.
+* Does not use AI.
 
 ## Synopsis
 
@@ -86,6 +87,7 @@ List all available recon workers:
 
 ```shell
 $ ronin-recon workers
+  api/crt_sh
   dns/lookup
   dns/mailservers
   dns/nameservers
@@ -95,7 +97,6 @@ $ ronin-recon workers
   dns/suffix_enum
   net/cert_enum
   net/cert_grab
-  net/cert_sh
   net/ip_range_enum
   net/port_scan
   net/service_id

data/data/completions/ronin-recon

@@ -11,7 +11,7 @@ _ronin-recon_completions_filter() {
 
   if [[ "${cur:0:1}" == "-" ]]; then
     echo "$words"
-  
+
   else
     for word in $words; do
       [[ "${word:0:1}" != "-" ]] && result+=("$word")
@@ -29,67 +29,67 @@ _ronin-recon_completions() {
 
   case "$compline" in
     'run'*'--config-file')
-      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -A file -- "$cur" )
+      while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -A file -- "$cur")
       ;;
 
     'run'*'--worker-file')
-      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -A file -- "$cur" )
+      while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -A file -- "$cur")
       ;;
 
     'worker'*'--file')
-      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -A file -- "$cur" )
+      while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -A file -- "$cur")
       ;;
 
     'run'*'--output')
-      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -A file -- "$cur" )
+      while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -A file -- "$cur")
       ;;
 
     'test'*'--file')
-      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -A file -- "$cur" )
+      while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -A file -- "$cur")
       ;;
 
     'completion'*)
-      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_ronin-recon_completions_filter "--print --install --uninstall")" -- "$cur" )
+      while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -W "$(_ronin-recon_completions_filter "--print --install --uninstall")" -- "$cur")
       ;;
 
     'worker'*'-f')
-      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -A file -- "$cur" )
+      while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -A file -- "$cur")
       ;;
 
     'test'*'-f')
-      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -A file -- "$cur" )
+      while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -A file -- "$cur")
       ;;
 
     'run'*'-C')
-      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -A file -- "$cur" )
+      while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -A file -- "$cur")
       ;;
 
     'run'*'-o')
-      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -A file -- "$cur" )
+      while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -A file -- "$cur")
       ;;
 
     'worker'*)
-      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_ronin-recon_completions_filter "--file -f --verbose -v")" -- "$cur" )
+      while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -W "$(_ronin-recon_completions_filter "--file -f --verbose -v")" -- "$cur")
       ;;
 
     'test'*)
-      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_ronin-recon_completions_filter "--file -f --debug -D --param -p")" -- "$cur" )
+      while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -W "$(_ronin-recon_completions_filter "--file -f --debug -D --param -p")" -- "$cur")
       ;;
 
     'new'*)
-      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_ronin-recon_completions_filter "--type -t --author -a --author-email -e --summary -S --description -D --reference -R --accepts -A --outputs -O --intensity -I")" -- "$cur" )
+      while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -W "$(_ronin-recon_completions_filter "--type -t --author -a --author-email -e --summary -S --description -D --reference -R --accepts -A --outputs -O --intensity -I")" -- "$cur")
       ;;
 
     'run'*)
-      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_ronin-recon_completions_filter "--debug -D --db --db-uri --db-file --config-file -C --worker -w --enable -e --disable -d --worker-file --param -p --concurrency -c --intensity --max-depth --output -o --output-format -F --import --ignore -I")" -- "$cur" )
+      while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -W "$(_ronin-recon_completions_filter "--debug -D --db --db-uri --db-file --config-file -C --worker -w --enable -e --disable -d --worker-file --param -p --concurrency -c --intensity --max-depth --output -o --output-format -F --import --ignore -I")" -- "$cur")
       ;;
 
     *)
-      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_ronin-recon_completions_filter "--version -V help completion irb new run test worker workers")" -- "$cur" )
+      while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -W "$(_ronin-recon_completions_filter "--version -V help completion irb new run test worker workers")" -- "$cur")
       ;;
 
   esac
 } &&
-complete -F _ronin-recon_completions ronin-recon
+  complete -F _ronin-recon_completions ronin-recon
 
 # ex: filetype=sh

data/gemspec.yml

@@ -45,13 +45,13 @@ dependencies:
   async-http: ~> 0.60
   wordlist: ~> 1.0, >= 1.0.3
   # Ronin dependencies:
-  ronin-support: ~> 1.1.0.rc1
-  ronin-core: ~> 0.2.0.rc1
-  ronin-db: ~> 0.2.0.rc1
+  ronin-support: ~> 1.1
+  ronin-core: ~> 0.2
+  ronin-db: ~> 0.2
   ronin-repos: ~> 0.1
-  ronin-masscan: ~> 0.1.0.rc1
-  ronin-nmap: ~> 0.1.0.rc1
-  ronin-web-spider: ~> 0.2.0.rc1
+  ronin-masscan: ~> 0.1
+  ronin-nmap: ~> 0.1
+  ronin-web-spider: ~> 0.2
 
 development_dependencies:
   bundler: ~> 2.0

data/lib/ronin/recon/builtin/api/crt_sh.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+#
+# ronin-recon - A micro-framework and tool for performing reconnaissance.
+#
+# Copyright (c) 2023-2024 Hal Brodigan (postmodern.mod3@gmail.com)
+#
+# ronin-recon is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# ronin-recon is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with ronin-recon.  If not, see <https://www.gnu.org/licenses/>.
+#
+
+require 'ronin/recon/worker'
+require 'ronin/support/text/patterns/network'
+
+require 'async/http/internet/instance'
+require 'set'
+
+module Ronin
+  module Recon
+    module API
+      #
+      # A recon worker that queries https://crt.sh and returns host from each
+      # domains certificate
+      #
+      class CrtSh < Worker
+
+        register 'api/crt_sh'
+
+        summary 'Queries https://crt.sh'
+
+        description <<~DESC
+          Queries https://crt.sh and returns the host names from each valid
+          certificate for the domain.
+        DESC
+
+        accepts Domain
+        outputs Host
+        intensity :passive
+        concurrency 1
+
+        # The HTTP client for `https://crt.sh`.
+        #
+        # @return [Async::HTTP::Client]
+        #
+        # @api private
+        attr_reader :client
+
+        #
+        # Initializes the `api/crt_sh` worker.
+        #
+        # @param [Hash{Symbol => Object}] kwargs
+        #   Additional keyword arguments.
+        #
+        # @api private
+        #
+        def initialize(**kwargs)
+          super(**kwargs)
+
+          @client = Async::HTTP::Client.new(
+            Async::HTTP::Endpoint.for('https','crt.sh')
+          )
+        end
+
+        # Regular expression to verify valid host names.
+        #
+        # @api private
+        HOST_NAME_REGEX = /\A#{Support::Text::Patterns::HOST_NAME}\z/
+
+        #
+        # Returns host from each domains certificate.
+        #
+        # @param [Values::Domain] domain
+        #   The domain value to check.
+        #
+        # @yield [host]
+        #   If the domain has certificates, then a host value will be
+        #   yielded.
+        #
+        # @yieldparam [Values::Host] host
+        #   The host from certificate.
+        #
+        def process(domain)
+          path      = "/?dNSName=#{domain}&exclude=expired&output=json"
+          response  = @client.get(path)
+          certs     = JSON.parse(response.read, symbolize_names: true)
+          hostnames = Set.new
+
+          certs.each do |cert|
+            common_name = cert[:common_name]
+
+            if common_name &&
+               common_name =~ HOST_NAME_REGEX &&
+               hostnames.add?(common_name)
+              yield Host.new(common_name)
+            end
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/ronin/recon/builtin/dns/subdomain_enum.rb

@@ -33,6 +33,7 @@ module Ronin
       #
       class SubdomainEnum < DNSWorker
 
+        # The path to the default common subdomains wordlist.
         DEFAULT_WORDLIST = File.join(WORDLISTS_DIR, 'subdomains-1000.txt.gz')
 
         register 'dns/subdomain_enum'

data/lib/ronin/recon/builtin/net/port_scan.rb

@@ -41,6 +41,7 @@ module Ronin
 
         accepts IP
         outputs OpenPort
+        concurrency 1 # prevents overloading the network interface
 
         param :ports, String, desc: 'Optional port list to scan'
 

data/lib/ronin/recon/builtin/net/service_id.rb

@@ -64,7 +64,7 @@ module Ronin
             else
               yield Website.http(open_port.host,open_port.number)
             end
-          when 'https'
+          when 'https', 'https-alt'
             yield Website.https(open_port.host,open_port.number)
           end
         end

data/lib/ronin/recon/builtin/ssl/cert_grab.rb

@@ -58,14 +58,22 @@ module Ronin
         #
         def process(open_port)
           if open_port.ssl?
+            context = OpenSSL::SSL::SSLContext.new
+
+            context.verify_mode = OpenSSL::SSL::VERIFY_NONE
+
             address  = open_port.address
             port     = open_port.number
-            endpoint = Async::IO::Endpoint.ssl(address,port)
+            endpoint = Async::IO::Endpoint.ssl(address,port, ssl_context: context)
 
-            endpoint.connect do |socket|
-              peer_cert = socket.peer_cert
+            begin
+              endpoint.connect do |socket|
+                peer_cert = socket.peer_cert
 
-              yield Cert.new(peer_cert)
+                yield Cert.new(peer_cert)
+              end
+            rescue OpenSSL::SSL::SSLError
+              # abort if we cannot successfully establish a SSL/TLS connection
             end
           end
         end

data/lib/ronin/recon/builtin/web/dir_enum.rb

@@ -69,6 +69,9 @@ module Ronin
         def process(website)
           wordlist = Wordlist.open(params[:wordlist] || DEFAULT_WORDLIST)
           queue    = Async::LimitedQueue.new(params[:concurrency])
+          endpoint = Async::HTTP::Endpoint.for(
+            website.scheme, website.host, port: website.port
+          )
           base_url = website.to_s
 
           Async do |task|
@@ -83,19 +86,23 @@ module Ronin
             # spawn the sub-tasks
             params[:concurrency].times do
               task.async do
-                http = Async::HTTP::Internet.instance
+                http = Async::HTTP::Client.new(endpoint)
 
                 while (dir = queue.dequeue)
                   path    = "/#{URI.encode_uri_component(dir)}"
-                  url     = "#{base_url}#{path}"
                   retries = 0
 
                   begin
-                    response = http.head(url)
+                    response = http.head(path)
+                    status   = response.status
 
-                    if VALID_STATUS_CODES.include?(response.status)
-                      yield URL.new(url, status:  response.status,
-                                         headers: response.headers)
+                    if VALID_STATUS_CODES.include?(status)
+                      headers = response.headers.to_h
+
+                      yield URL.new(
+                        "#{base_url}#{path}", status:  status,
+                                              headers: headers
+                      )
                     end
                   rescue Errno::ECONNREFUSED,
                          SocketError

data/lib/ronin/recon/builtin/web/email_addresses.rb

@@ -55,12 +55,17 @@ module Ronin
         #   Email address found on the page.
         #
         def process(url)
-          return nil unless url.body
+          if (body = url.body)
+            if body.encoding == Encoding::ASCII_8BIT
+              # forcibly convert and scrub binary data into UTF-8 data
+              body = body.dup
+              body.force_encoding(Encoding::UTF_8)
+              body.scrub!
+            end
 
-          email_pattern = Ronin::Support::Text::Patterns::EMAIL_ADDRESS
-
-          url.body.force_encoding(Encoding::UTF_8).scan(email_pattern) do |email|
-            yield EmailAddress.new(email)
+            body.scan(Support::Text::Patterns::EMAIL_ADDRESS) do |email|
+              yield EmailAddress.new(email)
+            end
           end
         end
 

data/lib/ronin/recon/builtin/web/spider.rb

@@ -32,10 +32,14 @@ module Ronin
 
         register 'web/spider'
 
-        summary 'Spiders a website'
+        summary 'Spiders a website and finds every URL'
 
         description <<~DESC
-          Spiders a website and returns every URL.
+          Spiders a website and finds every URL.
+
+          * Visits every `a`, `iframe`, `frame`, `link`, and `script` URL.
+          * Extracts paths from JavaScript.
+          * Extracts URLs from JavaScript.
         DESC
 
         accepts Website

data/lib/ronin/recon/cli/commands/run.rb

@@ -50,7 +50,7 @@ module Ronin
         #         --worker-file FILE           Loads a worker from a file
         #     -p, --param WORKER.NAME=VALUE    Sets a param for a worker
         #     -c, --concurrency WORKER=NUM     Sets the concurrency of a worker
-        #         --max-depth NUM              The maximum recon depth (Default: 3)
+        #         --max-depth NUM              The maximum recon depth (Default: 10)
         #     -o, --output FILE                The output file to write results to
         #     -I, --ignore VALUE               The values to ignore in result
         #     -F txt|list|csv|json|ndjson|dot|svg|png|pdf,
@@ -144,7 +144,7 @@ module Ronin
           option :max_depth, value: {
                                type:    Integer,
                                usage:   'NUM',
-                               default: 3
+                               default: 10
                              },
                              desc: 'The maximum recon depth'
 
@@ -226,7 +226,7 @@ module Ronin
 
           # The values that are out of scope.
           #
-          # @return [Array<Values::Value>]
+          # @return [Array<Value>]
           attr_reader :ignore
 
           #
@@ -318,7 +318,7 @@ module Ronin
           # @param [String] value
           #   The value to parse.
           #
-          # @return [Values::Value]
+          # @return [Value]
           #   The parsed value.
           #
           def parse_value(value)
@@ -393,7 +393,7 @@ module Ronin
           #
           # Imports a discovered value into ronin-db.
           #
-          # @param [Values::Value] value
+          # @param [Value] value
           #   A discovered recon value to import.
           #
           def import_value(value)
@@ -403,10 +403,10 @@ module Ronin
           #
           # Imports a connection between two values into ronin-db.
           #
-          # @param [Values::Value] value
+          # @param [Value] value
           #   A discovered recon value to import.
           #
-          # @param [Values::Value] parent
+          # @param [Value] parent
           #   The parent value of the discovered recon value.
           #
           def import_connection(value,parent)

data/lib/ronin/recon/cli/commands/test.rb

@@ -54,7 +54,7 @@ module Ronin
           include Core::CLI::Logging
           include Core::CLI::Options::Param
 
-          usage '[options] {IP | IP-range | DOMAIN | HOST | WILDCARD | WEBSITE}'
+          usage '[options] {--file FILE | NAME} {IP | IP-range | DOMAIN | HOST | WILDCARD | WEBSITE}'
 
           argument :value, required: true,
                            usage:    'IP|IP-range|DOMAIN|HOST|WILDCARD|WEBSITE',
@@ -70,6 +70,9 @@ module Ronin
           # @param [String, nil] name
           #   The optional worker name to load and print metadata for.
           #
+          # @param [String] value
+          #   The input value for the worker.
+          #
           def run(name=nil,value)
             super(name)
 

data/lib/ronin/recon/cli/printing.rb

@@ -33,17 +33,17 @@ module Ronin
         # Mapping of {Value} classes to printable names.
         VALUE_CLASS_NAMES = {
           Values::Domain       => 'domain',
+          Values::Mailserver   => 'mailserver',
+          Values::Nameserver   => 'nameserver',
+          Values::Wildcard     => 'wildcard host name',
           Values::Host         => 'host',
           Values::IP           => 'IP address',
           Values::IPRange      => 'IP range',
-          Values::Mailserver   => 'mailserver',
-          Values::Nameserver   => 'nameserver',
           Values::OpenPort     => 'open port',
           Values::Cert         => 'SSL/TLS certificate',
-          Values::EmailAddress => 'email addresse',
-          Values::URL          => 'URL',
           Values::Website      => 'website',
-          Values::Wildcard     => 'wildcard host name'
+          Values::URL          => 'URL',
+          Values::EmailAddress => 'email addresse'
         }
 
         #
@@ -66,7 +66,7 @@ module Ronin
         #
         # Formats a value object into a human readable string.
         #
-        # @param [Values::Value] value
+        # @param [Value] value
         #   The value object to format.
         #
         # @return [String]
@@ -77,17 +77,18 @@ module Ronin
         #
         def format_value(value)
           case value
-          when Values::Domain     then "domain #{value}"
-          when Values::Mailserver then "mailserver #{value}"
-          when Values::Nameserver then "nameserver #{value}"
-          when Values::Host       then "host #{value}"
-          when Values::IP         then "IP address #{value}"
-          when Values::IPRange    then "IP range #{value}"
-          when Values::OpenPort   then "open #{value.protocol.upcase} port #{value}"
-          when Values::Cert       then "SSL/TLS certificate #{value.subject}"
-          when Values::URL        then "URL #{value}"
-          when Values::Website    then "website #{value}"
-          when Values::Wildcard   then "wildcard host name #{value}"
+          when Values::Domain       then "domain #{value}"
+          when Values::Mailserver   then "mailserver #{value}"
+          when Values::Nameserver   then "nameserver #{value}"
+          when Values::Wildcard     then "wildcard host name #{value}"
+          when Values::Host         then "host #{value}"
+          when Values::IP           then "IP address #{value}"
+          when Values::IPRange      then "IP range #{value}"
+          when Values::OpenPort     then "open #{value.protocol.upcase} port #{value}"
+          when Values::Cert         then "SSL/TLS certificate #{value.subject}"
+          when Values::Website      then "website #{value}"
+          when Values::URL          then "URL #{value}"
+          when Values::EmailAddress then "email address #{value}"
           else
             raise(NotImplementedError,"value class #{value.class} not supported")
           end
@@ -96,7 +97,7 @@ module Ronin
         #
         # Prints a newly discovered value.
         #
-        # @param [Values::Value] value
+        # @param [Value] value
         #   The value to print.
         #
         # @param [Value, nil] parent

data/lib/ronin/recon/config.rb

@@ -72,6 +72,8 @@ module Ronin
 
         # The default workers configuration.
         DEFAULT = Set[
+          # NOTE: disabled due to rate limiting issues
+          # 'api/crt_sh',
           'dns/lookup',
           'dns/mailservers',
           'dns/nameservers',
@@ -84,8 +86,6 @@ module Ronin
           'net/service_id',
           'ssl/cert_grab',
           'ssl/cert_enum',
-          # NOTE: disabled due to rate limiting issues
-          # 'ssl/cert_sh',
           'web/dir_enum',
           'web/email_addresses',
           'web/spider'