ronin-web-spider 0.1.0.beta2 → 0.1.1

data/lib/ronin/web/spider/agent.rb

@@ -1,7 +1,8 @@
+# frozen_string_literal: true
 #
 # ronin-web-spider - A collection of common web spidering routines.
 #
-# Copyright (c) 2006-2022 Hal Brodigan (postmodern.mod3 at gmail.com)
+# Copyright (c) 2006-2023 Hal Brodigan (postmodern.mod3 at gmail.com)
 #
 # ronin-web-spider is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Lesser General Public License as published
@@ -122,6 +123,8 @@ module Ronin
         # The visited host names.
         #
         # @return [Set<String>, nil]
+        #
+        # @api public
         attr_reader :visited_hosts
 
         #
@@ -132,6 +135,13 @@ module Ronin
         #
         # @yieldparam [String] host
         #
+        # @example
+        #   spider.every_host do |host|
+        #     puts "Spidring #{host} ..."
+        #   end
+        #
+        # @api public
+        #
         def every_host
           @visited_hosts ||= Set.new
 
@@ -147,6 +157,8 @@ module Ronin
         # All certificates encountered while spidering.
         #
         # @return [Array<Ronin::Support::Crypto::Cert>]
+        #
+        # @api public
         attr_reader :collected_certs
 
         #
@@ -157,6 +169,13 @@ module Ronin
         #
         # @yieldparam [Ronin::Support::Crypto::Cert]
         #
+        # @example
+        #   spider.every_cert do |cert|
+        #     puts "Discovered new cert for #{cert.subject.command_name}, #{cert.subject_alt_name}"
+        #   end
+        #
+        # @api public
+        #
         def every_cert
           @collected_certs ||= []
 
@@ -185,8 +204,15 @@ module Ronin
         # @yieldparam [Spidr::Page] favicon
         #   An encountered `.ico` file.
         #
+        # @example
+        #   spider.every_favicon do |page|
+        #     # ...
+        #   end
+        #
         # @see https://rubydoc.info/gems/spidr/Spidr/Page
         #
+        # @api public
+        #
         def every_favicon
           every_page do |page|
             yield page if page.icon?
@@ -197,14 +223,23 @@ module Ronin
         # Passes every non-empty HTML comment to the given block.
         #
         # @yield [comment]
-        #   The given block will be passevery HTML comment.
+        #   The given block will be pass every HTML comment.
         #
         # @yieldparam [String] comment
         #   The HTML comment inner text, with leading and trailing whitespace
         #   stripped.
         #
+        # @example
+        #   spider.every_html_comment do |comment|
+        #     puts comment
+        #   end
+        #
+        # @api public
+        #
         def every_html_comment
           every_html_page do |page|
+            next unless page.doc
+
             page.doc.xpath('//comment()').each do |comment|
               comment_text = comment.inner_text.strip
 
@@ -224,24 +259,71 @@ module Ronin
         # @yieldparam [String] js
         #   The JavaScript source code.
         #
+        # @example
+        #   spider.every_javascript do |js|
+        #     puts js
+        #   end
+        #
+        # @api public
+        #
         def every_javascript
           # yield inner text of every `<script type="text/javascript">` tag
           # and every `.js` URL.
           every_html_page do |page|
+            next unless page.doc
+
             page.doc.xpath('//script[@type="text/javascript"]').each do |script|
-              unless script.inner_text.empty?
-                yield script.inner_text
+              source = script.inner_text
+              source.force_encoding(Encoding::UTF_8)
+
+              unless source.empty?
+                yield source
               end
             end
           end
 
           every_javascript_page do |page|
-            yield page.body
+            source = page.body
+            source.force_encoding(Encoding::UTF_8)
+
+            yield source
           end
         end
 
         alias every_js every_javascript
 
+        # Regex to match and skip JavaScript inline regexes.
+        #
+        # @api private
+        #
+        # @since 0.1.1
+        JAVASCRIPT_INLINE_REGEX = %r{
+          (?# match before the regex to avoid matching division operators )
+          (?:[\{\[\(;:,]\s*|=\s*)
+          /
+            (?# inline regex contents )
+            (?:
+              \[ (?:\\. | [^\]]) \] (?# [...] ) |
+              \\.                   (?# backslash escaped characters ) |
+              [^/]                  (?# everything else )
+            )+
+          /[dgimsuvy]* (?# also match any regex flags )
+        }mx
+
+        # Regex to match and skip JavaScript template literals.
+        #
+        # @note
+        #   This regex will not properly match nested template literals:
+        #
+        #   ```javascript
+        #   `foo ${`bar ${1+1}`}`
+        #   ```
+        #
+        # @api private
+        #
+        # @since 0.1.1
+        JAVASCRIPT_TEMPLATE_LITERAL = /`(?:\\`|[^`])+`/m
+
         #
         # Passes every JavaScript string value to the given block.
         #
@@ -252,10 +334,30 @@ module Ronin
         # @yieldparam [String] string
         #   The parsed contents of a JavaScript string.
         #
+        # @example
+        #   spider.every_javascript_string do |str|
+        #     puts str
+        #   end
+        #
+        # @api public
+        #
         def every_javascript_string
           every_javascript do |js|
-            js.scan(Support::Text::Patterns::STRING) do |js_string|
-              yield Support::Encoding::JS.unquote(js_string)
+            scanner = StringScanner.new(js)
+
+            until scanner.eos?
+              # NOTE: this is a naive JavaScript string scanner and should
+              # eventually be replaced with a real JavaScript lexer or parser.
+              case scanner.peek(1)
+              when '"', "'" # beginning of a quoted string
+                js_string = scanner.scan(Support::Text::Patterns::STRING)
+
+                yield Support::Encoding::JS.unquote(js_string)
+              else
+                scanner.skip(JAVASCRIPT_INLINE_REGEX) ||
+                  scanner.skip(JAVASCRIPT_TEMPLATE_LITERAL) ||
+                  scanner.getch
+              end
             end
           end
         end
@@ -271,6 +373,13 @@ module Ronin
         # @yieldparam [String] comment
         #   The contents of a JavaScript comment.
         #
+        # @example
+        #   spider.every_javascript_comment do |comment|
+        #     puts comment
+        #   end
+        #
+        # @api public
+        #
         def every_javascript_comment(&block)
           every_javascript do |js|
             js.scan(Support::Text::Patterns::JAVASCRIPT_COMMENT,&block)
@@ -288,9 +397,16 @@ module Ronin
         # @yieldparam [String] comment
         #   The contents of a HTML or JavaScript comment.
         #
+        # @example
+        #   spider.every_comment do |comment|
+        #     puts comment
+        #   end
+        #
         # @see #every_html_comment
         # @see #every_javascript_comment
         #
+        # @api public
+        #
         def every_comment(&block)
           every_html_comment(&block)
           every_javascript_comment(&block)

data/lib/ronin/web/spider/archive.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 #
 # ronin-web-spider - A collection of common web spidering routines.
 #
@@ -29,6 +30,9 @@ module Ronin
       #
       # Spider a host and archive every web page:
       #
+      #     require 'ronin/web/spider'
+      #     require 'ronin/web/spider/archive'
+      #
       #     Ronin::Web::Spider::Archive.open('path/to/root') do |archive|
       #       Ronin::Web::Spider.every_page(host: 'example.com') do |page|
       #         archive.write(page.url,page.body)

data/lib/ronin/web/spider/exceptions.rb

@@ -1,7 +1,8 @@
+# frozen_string_literal: true
 #
 # ronin-web-spider - A collection of common web spidering routines.
 #
-# Copyright (c) 2006-2022 Hal Brodigan (postmodern.mod3 at gmail.com)
+# Copyright (c) 2006-2023 Hal Brodigan (postmodern.mod3 at gmail.com)
 #
 # ronin-web-spider is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Lesser General Public License as published

data/lib/ronin/web/spider/git_archive.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 #
 # ronin-web-spider - A collection of common web spidering routines.
 #
@@ -30,10 +31,10 @@ module Ronin
       #
       # Spider a host and archive every web page to a Git repository:
       #
-      #     require 'ronin/web/spider/git_archive'
       #     require 'ronin/web/spider'
+      #     require 'ronin/web/spider/git_archive'
       #     require 'date'
-      #     
+      #
       #     Ronin::Web::Spider::GitArchive.open('path/to/root') do |archive|
       #       archive.commit("Updated #{Date.today}") do
       #         Ronin::Web::Spider.every_page(host: 'example.com') do |page|

data/lib/ronin/web/spider/version.rb

@@ -1,7 +1,8 @@
+# frozen_string_literal: true
 #
 # ronin-web-spider - A collection of common web spidering routines.
 #
-# Copyright (c) 2006-2022 Hal Brodigan (postmodern.mod3 at gmail.com)
+# Copyright (c) 2006-2023 Hal Brodigan (postmodern.mod3 at gmail.com)
 #
 # ronin-web-spider is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Lesser General Public License as published
@@ -21,7 +22,7 @@ module Ronin
   module Web
     module Spider
       # ronin-web-spider version
-      VERSION = '0.1.0.beta2'
+      VERSION = '0.1.1'
     end
   end
 end

data/lib/ronin/web/spider.rb

@@ -1,7 +1,8 @@
+# frozen_string_literal: true
 #
 # ronin-web-spider - A collection of common web spidering routines.
 #
-# Copyright (c) 2006-2022 Hal Brodigan (postmodern.mod3 at gmail.com)
+# Copyright (c) 2006-2023 Hal Brodigan (postmodern.mod3 at gmail.com)
 #
 # ronin-web-spider is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Lesser General Public License as published
@@ -22,6 +23,286 @@ require 'ronin/web/spider/version'
 
 module Ronin
   module Web
+    #
+    # A collection of common web spidering routines using the [spidr] gem.
+    #
+    # [spidr]: https://github.com/postmodern/spidr#readme
+    #
+    # ## Examples
+    #
+    # Spider a host:
+    #
+    # ```ruby
+    # require 'ronin/web/spider'
+    #
+    # Ronin::Web::Spider.start_at('http://tenderlovemaking.com/') do |agent|
+    #   # ...
+    # end
+    # ```
+    #
+    # Spider a host:
+    #
+    # ```ruby
+    # Ronin::Web::Spider.host('solnic.eu') do |agent|
+    #   # ...
+    # end
+    # ```
+    #
+    # Spider a domain (and any sub-domains):
+    #
+    # ```ruby
+    # Ronin::Web::Spider.domain('ruby-lang.org') do |agent|
+    #   # ...
+    # end
+    # ```
+    #
+    # Spider a site:
+    #
+    # ```ruby
+    # Ronin::Web::Spider.site('http://www.rubyflow.com/') do |agent|
+    #   # ...
+    # end
+    # ```
+    #
+    # Spider multiple hosts:
+    #
+    # ```ruby
+    # Ronin::Web::Spider.start_at('http://company.com/', hosts: ['company.com', /host[\d]+\.company\.com/]) do |agent|
+    #   # ...
+    # end
+    # ```
+    #
+    # Do not spider certain links:
+    #
+    # ```ruby
+    # Ronin::Web::Spider.site('http://company.com/', ignore_links: [%{^/blog/}]) do |agent|
+    #   # ...
+    # end
+    # ```
+    #
+    # Do not spider links on certain ports:
+    #
+    # ```ruby
+    # Ronin::Web::Spider.site('http://company.com/', ignore_ports: [8000, 8010, 8080]) do |agent|
+    #   # ...
+    # end
+    # ```
+    #
+    # Do not spider links blacklisted in robots.txt:
+    #
+    # ```ruby
+    # Ronin::Web::Spider.site('http://company.com/', robots: true) do |agent|
+    #   # ...
+    # end
+    # ```
+    #
+    # Print out visited URLs:
+    #
+    # ```ruby
+    # Ronin::Web::Spider.site('http://www.rubyinside.com/') do |spider|
+    #   spider.every_url { |url| puts url }
+    # end
+    # ```
+    #
+    # Build a URL map of a site:
+    #
+    # ```ruby
+    # url_map = Hash.new { |hash,key| hash[key] = [] }
+    #
+    # Ronin::Web::Spider.site('http://intranet.com/') do |spider|
+    #   spider.every_link do |origin,dest|
+    #     url_map[dest] << origin
+    #   end
+    # end
+    # ```
+    #
+    # Print out the URLs that could not be requested:
+    #
+    # ```ruby
+    # Ronin::Web::Spider.site('http://company.com/') do |spider|
+    #   spider.every_failed_url { |url| puts url }
+    # end
+    # ```
+    #
+    # Finds all pages which have broken links:
+    #
+    # ```ruby
+    # url_map = Hash.new { |hash,key| hash[key] = [] }
+    #
+    # spider = Ronin::Web::Spider.site('http://intranet.com/') do |spider|
+    #   spider.every_link do |origin,dest|
+    #     url_map[dest] << origin
+    #   end
+    # end
+    #
+    # spider.failures.each do |url|
+    #   puts "Broken link #{url} found in:"
+    #
+    #   url_map[url].each { |page| puts "  #{page}" }
+    # end
+    # ```
+    #
+    # Search HTML and XML pages:
+    #
+    # ```ruby
+    # Ronin::Web::Spider.site('http://company.com/') do |spider|
+    #   spider.every_page do |page|
+    #     puts ">>> #{page.url}"
+    #
+    #     page.search('//meta').each do |meta|
+    #       name = (meta.attributes['name'] || meta.attributes['http-equiv'])
+    #       value = meta.attributes['content']
+    #
+    #       puts "  #{name} = #{value}"
+    #     end
+    #   end
+    # end
+    # ```
+    #
+    # Print out the titles from every page:
+    #
+    # ```ruby
+    # Ronin::Web::Spider.site('https://www.ruby-lang.org/') do |spider|
+    #   spider.every_html_page do |page|
+    #     puts page.title
+    #   end
+    # end
+    # ```
+    #
+    # Print out every HTTP redirect:
+    #
+    # ```ruby
+    # Ronin::Web::Spider.host('company.com') do |spider|
+    #   spider.every_redirect_page do |page|
+    #     puts "#{page.url} -> #{page.headers['Location']}"
+    #   end
+    # end
+    # ```
+    #
+    # Find what kinds of web servers a host is using, by accessing the headers:
+    #
+    # ```ruby
+    # servers = Set[]
+    #
+    # Ronin::Web::Spider.host('company.com') do |spider|
+    #   spider.all_headers do |headers|
+    #     servers << headers['server']
+    #   end
+    # end
+    # ```
+    #
+    # Pause the spider on a forbidden page:
+    #
+    # ```ruby
+    # Ronin::Web::Spider.host('company.com') do |spider|
+    #   spider.every_forbidden_page do |page|
+    #     spider.pause!
+    #   end
+    # end
+    # ```
+    #
+    # Skip the processing of a page:
+    #
+    # ```ruby
+    # Ronin::Web::Spider.host('company.com') do |spider|
+    #   spider.every_missing_page do |page|
+    #     spider.skip_page!
+    #   end
+    # end
+    # ```
+    #
+    # Skip the processing of links:
+    #
+    # ```ruby
+    # Ronin::Web::Spider.host('company.com') do |spider|
+    #   spider.every_url do |url|
+    #     if url.path.split('/').find { |dir| dir.to_i > 1000 }
+    #       spider.skip_link!
+    #     end
+    #   end
+    # end
+    # ```
+    #
+    # Detect when a new host name is spidered:
+    #
+    # ```ruby
+    # Ronin::Web::Spider.domain('example.com') do |spider|
+    #   spider.every_host do |host|
+    #     puts "Spidring #{host} ..."
+    #   end
+    # end
+    # ```
+    #
+    # Detect when a new SSL/TLS certificate is encountered:
+    #
+    # ```ruby
+    # Ronin::Web::Spider.domain('example.com') do |spider|
+    #   spider.every_cert do |cert|
+    #     puts "Discovered new cert for #{cert.subject.command_name}, #{cert.subject_alt_name}"
+    #   end
+    # end
+    # ```
+    #
+    # Print the MD5 checksum of every `favicon.ico` file:
+    #
+    # ```ruby
+    # Ronin::Web::Spider.domain('example.com') do |spider|
+    #   spider.every_favicon do |page|
+    #     puts "#{page.url}: #{page.body.md5}"
+    #   end
+    # end
+    # ```
+    #
+    # Print every HTML comment:
+    #
+    # ```ruby
+    # Ronin::Web::Spider.domain('example.com') do |spider|
+    #   spider.every_html_comment do |comment|
+    #     puts comment
+    #   end
+    # end
+    # ```
+    #
+    # Print all JavaScript source code:
+    #
+    # ```ruby
+    # Ronin::Web::Spider.domain('example.com') do |spider|
+    #   spider.every_javascript do |js|
+    #     puts js
+    #   end
+    # end
+    # ```
+    #
+    # Print every JavaScript string literal:
+    #
+    # ```ruby
+    # Ronin::Web::Spider.domain('example.com') do |spider|
+    #   spider.every_javascript_string do |str|
+    #     puts str
+    #   end
+    # end
+    # ```
+    #
+    # Print every JavaScript comment:
+    #
+    # ```ruby
+    # Ronin::Web::Spider.domain('example.com') do |spider|
+    #   spider.every_javascript_comment do |comment|
+    #     puts comment
+    #   end
+    # end
+    # ```
+    #
+    # Print every HTML and JavaScript comment:
+    #
+    # ```ruby
+    # Ronin::Web::Spider.domain('example.com') do |spider|
+    #   spider.every_comment do |comment|
+    #     puts comment
+    #   end
+    # end
+    # ```
+    #
     module Spider
       #
       # Creates a new agent and begin spidering at the given URL.
@@ -41,6 +322,8 @@ module Ronin
       #
       # @see https://rubydoc.info/gems/spidr/Spidr/Agent#start_at-class_method
       #
+      # @api public
+      #
       def self.start_at(url,**kwargs,&block)
         Agent.start_at(url,**kwargs,&block)
       end
@@ -63,6 +346,8 @@ module Ronin
       #
       # @see https://rubydoc.info/gems/spidr/Spidr/Agent#host-class_method
       #
+      # @api public
+      #
       def self.host(name,**kwargs,&block)
         Agent.host(name,**kwargs,&block)
       end
@@ -85,6 +370,8 @@ module Ronin
       #
       # @see https://rubydoc.info/gems/spidr/Spidr/Agent#site-class_method
       #
+      # @api public
+      #
       def self.site(url,**kwargs,&block)
         Agent.site(url,**kwargs,&block)
       end
@@ -107,6 +394,8 @@ module Ronin
       #
       # @see https://rubydoc.info/gems/spidr/Spidr/Agent#domain-class_method
       #
+      # @api public
+      #
       def self.domain(name,**kwargs,&block)
         Agent.domain(name,**kwargs,&block)
       end

data/ronin-web-spider.gemspec

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# frozen_string_literal: true
 
 require 'yaml'
 
@@ -22,18 +22,19 @@ Gem::Specification.new do |gem|
   gem.homepage    = gemspec['homepage']
   gem.metadata    = gemspec['metadata'] if gemspec['metadata']
 
-  glob = lambda { |patterns| gem.files & Dir[*patterns] }
+  glob = ->(patterns) { gem.files & Dir[*patterns] }
 
   gem.files  = `git ls-files`.split($/)
   gem.files  = glob[gemspec['files']] if gemspec['files']
   gem.files += Array(gemspec['generated_files'])
+  # exclude test files from the packages gem
+  gem.files -= glob[gemspec['test_files'] || 'spec/{**/}*']
 
   gem.executables = gemspec.fetch('executables') do
     glob['bin/*'].map { |path| File.basename(path) }
   end
 
   gem.extensions       = glob[gemspec['extensions'] || 'ext/**/extconf.rb']
-  gem.test_files       = glob[gemspec['test_files'] || 'spec/{**/}*_spec.rb']
   gem.extra_rdoc_files = glob[gemspec['extra_doc_files'] || '*.{txt,md}']
 
   gem.require_paths = Array(gemspec.fetch('require_paths') {
@@ -45,7 +46,7 @@ Gem::Specification.new do |gem|
   gem.required_rubygems_version = gemspec['required_rubygems_version']
   gem.post_install_message      = gemspec['post_install_message']
 
-  split = lambda { |string| string.split(/,\s*/) }
+  split = ->(string) { string.split(/,\s*/) }
 
   if gemspec['dependencies']
     gemspec['dependencies'].each do |name,versions|