ronin-web-spider 0.1.1 → 0.2.0.rc2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dab34842325a731e13f23303b1dec66c7ab9d78b4805b982d518ba01024c9352
-  data.tar.gz: '0668f126b3e828c6409cc7b7adef2d74cd527f48de334b10a3d54f3767fe3afd'
+  metadata.gz: 7aedd94cd4b3f96a07824419722763a91374590e6944ab3ae58c6ff1432837ca
+  data.tar.gz: 67d2d63b5468838f60f8b3aa3975dcb8f0842f66ca1d776384fb29c6a7f6a8ca
 SHA512:
-  metadata.gz: d474705a601b7fe27be2a9c5f5e5485ed39b38dec1581db295b0e1ff524c987c9e74522afd24569829bc7e64f6930767104f9bd927a3007a17f627de003492f7
-  data.tar.gz: 397b84308ec62d51e1dba64cff37c75eda8ebd8c2e6a792487468158fb774aae566b8be5cd0388521774d62873e5ba6b751423bc1138ac9e3d47804bdd877a81
+  metadata.gz: b9e7588f16084226b812db561261122af81fea2bd2d4c1b529c7cc1763e48060c90f716475e930029159f9cd1f886866f47ea03be8c4131c850af0b6d73da8c0
+  data.tar.gz: 253283e0e5f8046d4d41fb9bf445933017e81b92e72c817aa0d4c6850c4952e0ec687e727196707f71e0343a97ba0f492a409e070a3c00e8d0fd8640feda75d5

data/ChangeLog.md

@@ -1,3 +1,18 @@
+### 0.2.0 / 2024-XX-XX
+
+* Added {Ronin::Web::Spider::Agent#every_javascript_url_string}.
+* Added {Ronin::Web::Spider::Agent#every_javascript_relative_path_string}.
+* Added {Ronin::Web::Spider::Agent#every_javascript_absolute_path_string}.
+* Added {Ronin::Web::Spider::Agent#every_javascript_path_string}.
+* Allow {Ronin::Web::Spider::Agent#every_html_comment},
+  {Ronin::Web::Spider::Agent#every_javascript every_javascript},
+  {Ronin::Web::Spider::Agent#every_javascript_string every_javascript_string},
+  {Ronin::Web::Spider::Agent#every_javascript_relative_path_string every_javascript_relative_path_string},
+  {Ronin::Web::Spider::Agent#every_javascript_absolute_path_string every_javascript_absolute_path_string},
+  {Ronin::Web::Spider::Agent#every_javascript_url_string every_javascript_url_string}, and
+  {Ronin::Web::Spider::Agent#every_javascript_comment every_javascript_comment}
+  to also yield a `Spidr::Page` block argument for additional context.
+
 ### 0.1.1 / 2024-06-19
 
 * Fixed {Ronin::Web::Spider::Agent#every_html_comment} and

data/README.md

@@ -304,6 +304,16 @@ Ronin::Web::Spider.domain('example.com') do |spider|
 end
 ```
 
+Print every JavaScript URL string literal:
+
+```ruby
+Ronin::Web::Spider.domain('example.com') do |spider|
+  spider.every_javascript_url_string do |url|
+    puts url
+  end
+end
+```
+
 Print every JavaScript comment:
 
 ```ruby
@@ -390,7 +400,7 @@ gem.add_dependency 'ronin-web-spider', '~> 0.1'
 
 ## License
 
-Copyright (c) 2006-2023 Hal Brodigan (postmodern.mod3 at gmail.com)
+Copyright (c) 2006-2024 Hal Brodigan (postmodern.mod3 at gmail.com)
 
 ronin-web-spider is free software: you can redistribute it and/or modify
 it under the terms of the GNU Lesser General Public License as published

data/lib/ronin/web/spider/agent.rb

@@ -2,7 +2,7 @@
 #
 # ronin-web-spider - A collection of common web spidering routines.
 #
-# Copyright (c) 2006-2023 Hal Brodigan (postmodern.mod3 at gmail.com)
+# Copyright (c) 2006-2024 Hal Brodigan (postmodern.mod3 at gmail.com)
 #
 # ronin-web-spider is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Lesser General Public License as published
@@ -23,6 +23,7 @@ require 'spidr/agent'
 require 'ronin/support/network/http'
 require 'ronin/support/crypto/cert'
 require 'ronin/support/text/patterns/source_code'
+require 'ronin/support/text/patterns/network'
 require 'ronin/support/encoding/js'
 
 module Ronin
@@ -225,10 +226,17 @@ module Ronin
         # @yield [comment]
         #   The given block will be pass every HTML comment.
         #
+        # @yield [comment, page]
+        #   If the block accepts two arguments, the HTML comment and the page
+        #   that the comment was found on will be passed to the given block.
+        #
         # @yieldparam [String] comment
         #   The HTML comment inner text, with leading and trailing whitespace
         #   stripped.
         #
+        # @yieldparam [Spidr::Page] page
+        #   The page that the HTML comment exists on.
+        #
         # @example
         #   spider.every_html_comment do |comment|
         #     puts comment
@@ -236,7 +244,7 @@ module Ronin
         #
         # @api public
         #
-        def every_html_comment
+        def every_html_comment(&block)
           every_html_page do |page|
             next unless page.doc
 
@@ -244,7 +252,11 @@ module Ronin
               comment_text = comment.inner_text.strip
 
               unless comment_text.empty?
-                yield comment_text
+                if block.arity == 2
+                  yield comment_text, page
+                else
+                  yield comment_text
+                end
               end
             end
           end
@@ -256,9 +268,17 @@ module Ronin
         # @yield [js]
         #   The given block will be passed every piece of JavaScript source.
         #
+        # @yield [js, page]
+        #   If the block accepts two arguments, the JavaScript source and the
+        #   page that the JavaScript source was found on will be passed to the
+        #   given block.
+        #
         # @yieldparam [String] js
         #   The JavaScript source code.
         #
+        # @yieldparam [Spidr::Page] page
+        #   The page that the JavaScript source was found in or on.
+        #
         # @example
         #   spider.every_javascript do |js|
         #     puts js
@@ -266,7 +286,7 @@ module Ronin
         #
         # @api public
         #
-        def every_javascript
+        def every_javascript(&block)
           # yield inner text of every `<script type="text/javascript">` tag
           # and every `.js` URL.
           every_html_page do |page|
@@ -277,7 +297,11 @@ module Ronin
               source.force_encoding(Encoding::UTF_8)
 
               unless source.empty?
-                yield source
+                if block.arity == 2
+                  yield source, page
+                else
+                  yield source
+                end
               end
             end
           end
@@ -286,7 +310,11 @@ module Ronin
             source = page.body
             source.force_encoding(Encoding::UTF_8)
 
-            yield source
+            if block.arity == 2
+              yield source, page
+            else
+              yield source
+            end
           end
         end
 
@@ -297,7 +325,7 @@ module Ronin
         # @api private
         #
         # @since 0.1.1
-        JAVASCRIPT_INLINE_REGEX = %r{
+        JAVASCRIPT_INLINE_REGEX_REGEX = %r{
           (?# match before the regex to avoid matching division operators )
           (?:[\{\[\(;:,]\s*|=\s*)
           /
@@ -322,7 +350,7 @@ module Ronin
         # @api private
         #
         # @since 0.1.1
-        JAVASCRIPT_TEMPLATE_LITERAL = /`(?:\\`|[^`])+`/m
+        JAVASCRIPT_TEMPLATE_LITERAL_REGEX = /`(?:\\`|[^`])+`/m
 
         #
         # Passes every JavaScript string value to the given block.
@@ -331,9 +359,17 @@ module Ronin
         #   The given block will be passed each JavaScript string with the quote
         #   marks removed.
         #
+        # @yield [string, page]
+        #   If the block accepts two arguments, the JavaScript string and the
+        #   page that the JavaScript string was found on will be passed to the
+        #   given block.
+        #
         # @yieldparam [String] string
         #   The parsed contents of a JavaScript string.
         #
+        # @yieldparam [Spidr::Page] page
+        #   The page that the JavaScript string was found in or on.
+        #
         # @example
         #   spider.every_javascript_string do |str|
         #     puts str
@@ -341,8 +377,8 @@ module Ronin
         #
         # @api public
         #
-        def every_javascript_string
-          every_javascript do |js|
+        def every_javascript_string(&block)
+          every_javascript do |js,page|
             scanner = StringScanner.new(js)
 
             until scanner.eos?
@@ -351,11 +387,16 @@ module Ronin
               case scanner.peek(1)
               when '"', "'" # beginning of a quoted string
                 js_string = scanner.scan(Support::Text::Patterns::STRING)
+                string    = Support::Encoding::JS.unquote(js_string)
 
-                yield Support::Encoding::JS.unquote(js_string)
+                if block.arity == 2
+                  yield string, page
+                else
+                  yield string
+                end
               else
-                scanner.skip(JAVASCRIPT_INLINE_REGEX) ||
-                  scanner.skip(JAVASCRIPT_TEMPLATE_LITERAL) ||
+                scanner.skip(JAVASCRIPT_INLINE_REGEX_REGEX) ||
+                  scanner.skip(JAVASCRIPT_TEMPLATE_LITERAL_REGEX) ||
                   scanner.getch
               end
             end
@@ -364,15 +405,215 @@ module Ronin
 
         alias every_js_string every_javascript_string
 
+        # Regular expression that matches relative paths within JavaScript.
+        #
+        # @note
+        #   This matches `foo/bar`, `foo/bar.ext`, `../foo`, and `foo.ext`,
+        #   but *not* `/foo`, `foo`, or `foo.`.
+        #
+        # @api private
+        #
+        # @since 0.2.0
+        JAVASCRIPT_RELATIVE_PATH_REGEX = %r{
+          \A
+            (?:
+               [^/\\. ]+\.[a-z0-9]+ (?# filename.ext)
+               |
+               [^/\\ ]+(?:/[^/\\ ]+)+ (?# dir/filename or dir/filename.ext)
+            )
+          \z
+        }x
+
+        #
+        # Passes every JavaScript relative path string to the given block.
+        #
+        # @yield [string]
+        #   The given block will be passed each JavaScript relative path string
+        #   with the quote marks removed.
+        #
+        # @yield [string, page]
+        #   If the block accepts two arguments, the JavaScript relative path
+        #   string and the page that the JavaScript relative path string was
+        #   found on will be passed to the given block.
+        #
+        # @yieldparam [String] string
+        #   The parsed contents of a literal JavaScript relative path string.
+        #
+        # @yieldparam [Spidr::Page] page
+        #   The page that the JavaScript relative path string was found in or
+        #   on.
+        #
+        # @example
+        #   spider.every_javascript_relative_path_string do |relative_path|
+        #     puts relative_path
+        #   end
+        #
+        # @api public
+        #
+        # @since 0.2.0
+        #
+        def every_javascript_relative_path_string(&block)
+          every_javascript_string do |string,page|
+            if string =~ JAVASCRIPT_RELATIVE_PATH_REGEX
+              if block.arity == 2
+                yield string, page
+              else
+                yield string
+              end
+            end
+          end
+        end
+
+        alias every_js_relative_path_string every_javascript_relative_path_string
+
+        # Regular expression that matches absolute paths within JavaScript.
+        #
+        # @api private
+        #
+        # @since 0.2.0
+        JAVASCRIPT_ABSOLUTE_PATH_REGEX = %r{\A(?:/[^/\\ ]+)+\z}
+
+        #
+        # Passes every JavaScript absolute path string to the given block.
+        #
+        # @yield [string]
+        #   The given block will be passed each JavaScript absolute path string
+        #   with the quote marks removed.
+        #
+        # @yield [string, page]
+        #   If the block accepts two arguments, the JavaScript absolute path
+        #   string and the page that the JavaScript absolute path string was
+        #   found on will be passed to the given block.
+        #
+        # @yieldparam [String] string
+        #   The parsed contents of a literal JavaScript absolute path string.
+        #
+        # @yieldparam [Spidr::Page] page
+        #   The page that the JavaScript absolute path string was found in or
+        #   on.
+        #
+        # @example
+        #   spider.every_javascript_absolute_path_string do |absolute_path|
+        #     puts absolute_path
+        #   end
+        #
+        # @api public
+        #
+        # @since 0.2.0
+        #
+        def every_javascript_absolute_path_string(&block)
+          every_javascript_string do |string,page|
+            if string =~ JAVASCRIPT_ABSOLUTE_PATH_REGEX
+              if block.arity == 2
+                yield string, page
+              else
+                yield string
+              end
+            end
+          end
+        end
+
+        alias every_js_absolute_path_string every_javascript_absolute_path_string
+
+        #
+        # Passes every JavaScript path string to the given block.
+        #
+        # @yield [string]
+        #   The given block will be passed each JavaScript path string with the
+        #   quote marks removed.
+        #
+        # @yield [string, page]
+        #   If the block accepts two arguments, the JavaScript path string and
+        #   the page that the JavaScript path string was found on will be
+        #   passed to the given block.
+        #
+        # @yieldparam [String] string
+        #   The parsed contents of a literal JavaScript path string.
+        #
+        # @yieldparam [Spidr::Page] page
+        #   The page that the JavaScript path string was found in or on.
+        #
+        # @example
+        #   spider.every_javascript_path_string do |path|
+        #     puts path
+        #   end
+        #
+        # @api public
+        #
+        # @since 0.2.0
+        #
+        def every_javascript_path_string(&block)
+          every_javascript_relative_path_string(&block)
+          every_javascript_absolute_path_string(&block)
+        end
+
+        alias every_js_path_string every_javascript_path_string
+
+        # Regular expression for identifying URLs.
+        #
+        # @api private
+        #
+        # @since 0.2.0
+        URL_REGEX = /\A#{Support::Text::Patterns::URL}\z/
+
+        #
+        # Passes every JavaScript URL string to the given block.
+        #
+        # @yield [string]
+        #   The given block will be passed each JavaScript URL string with the
+        #   quote marks removed.
+        #
+        # @yield [string, page]
+        #   If the block accepts two arguments, the JavaScript URL string and
+        #   the page that the JavaScript URL string was found on will be passed
+        #   to the given block.
+        #
+        # @yieldparam [String] string
+        #   The parsed contents of a literal JavaScript URL string.
+        #
+        # @yieldparam [Spidr::Page] page
+        #   The page that the JavaScript URL string was found in or on.
+        #
+        # @example
+        #   spider.every_javascript_url_string do |url|
+        #     puts url
+        #   end
+        #
+        # @api public
+        #
+        # @since 0.2.0
+        #
+        def every_javascript_url_string(&block)
+          every_javascript_string do |string,page|
+            if string =~ URL_REGEX
+              if block.arity == 2
+                yield string, page
+              else
+                yield string
+              end
+            end
+          end
+        end
+
+        alias every_js_url_string every_javascript_url_string
+
         #
         # Passes every JavaScript comment to the given block.
         #
         # @yield [comment]
         #   The given block will be passed each JavaScript comment.
         #
+        # @yield [comment, page]
+        #   If the block accepts two arguments, the JavaScript comment and the
+        #   page that the JavaScript comment was found on will be passed to the
+        #   given block.
+        #
         # @yieldparam [String] comment
         #   The contents of a JavaScript comment.
         #
+        # @yieldparam [Spidr::Page] page
+        #   The page that the JavaScript comment was found in or on.
+        #
         # @example
         #   spider.every_javascript_comment do |comment|
         #     puts comment
@@ -381,8 +622,14 @@ module Ronin
         # @api public
         #
         def every_javascript_comment(&block)
-          every_javascript do |js|
-            js.scan(Support::Text::Patterns::JAVASCRIPT_COMMENT,&block)
+          every_javascript do |js,page|
+            js.scan(Support::Text::Patterns::JAVASCRIPT_COMMENT) do |comment|
+              if block.arity == 2
+                yield comment, page
+              else
+                yield comment
+              end
+            end
           end
         end
 
@@ -394,9 +641,17 @@ module Ronin
         # @yield [comment]
         #   The given block will be passed each HTML or JavaScript comment.
         #
+        # @yield [comment, page]
+        #   If the block accepts two arguments, the HTML or JavaScript comment
+        #   and the page that the HTML/JavaScript comment was found on will be
+        #   passed to the given block.
+        #
         # @yieldparam [String] comment
         #   The contents of a HTML or JavaScript comment.
         #
+        # @yieldparam [Spidr::Page] page
+        #   The page that the HTML or JavaScript comment was found in or on.
+        #
         # @example
         #   spider.every_comment do |comment|
         #     puts comment

data/lib/ronin/web/spider/exceptions.rb

@@ -2,7 +2,7 @@
 #
 # ronin-web-spider - A collection of common web spidering routines.
 #
-# Copyright (c) 2006-2023 Hal Brodigan (postmodern.mod3 at gmail.com)
+# Copyright (c) 2006-2024 Hal Brodigan (postmodern.mod3 at gmail.com)
 #
 # ronin-web-spider is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Lesser General Public License as published

data/lib/ronin/web/spider/version.rb

@@ -2,7 +2,7 @@
 #
 # ronin-web-spider - A collection of common web spidering routines.
 #
-# Copyright (c) 2006-2023 Hal Brodigan (postmodern.mod3 at gmail.com)
+# Copyright (c) 2006-2024 Hal Brodigan (postmodern.mod3 at gmail.com)
 #
 # ronin-web-spider is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Lesser General Public License as published
@@ -22,7 +22,7 @@ module Ronin
   module Web
     module Spider
       # ronin-web-spider version
-      VERSION = '0.1.1'
+      VERSION = '0.2.0.rc2'
     end
   end
 end

data/lib/ronin/web/spider.rb

@@ -2,7 +2,7 @@
 #
 # ronin-web-spider - A collection of common web spidering routines.
 #
-# Copyright (c) 2006-2023 Hal Brodigan (postmodern.mod3 at gmail.com)
+# Copyright (c) 2006-2024 Hal Brodigan (postmodern.mod3 at gmail.com)
 #
 # ronin-web-spider is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Lesser General Public License as published

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ronin-web-spider
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0.rc2
 platform: ruby
 authors:
 - Postmodern
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-06-20 00:00:00.000000000 Z
+date: 2024-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: spidr