ronin-web-spider 0.1.0.beta1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f4a08a150f4301148cc86877702ea981f2083d173c3101ee6047d44cfc5f076c
-  data.tar.gz: 8fb751613e0cee2161d08b338cce434b252ba83f62971275b5a33d0060cec567
+  metadata.gz: fcb3d69132ae37799758c37282083f3b876e04e76aa3ab9f500f251b7df0984d
+  data.tar.gz: 04b92b26f1bcd6166530ddfe225cde18a4bbaa8a1eb3b395120ae1e6b41aec4b
 SHA512:
-  metadata.gz: 5af53f0fbf55f13d8ccd2ff9fd13d7cfba7d6ce7c8aae90e5b44a467617212eb77247f9e343da118bdc8a369a9cbb7d240f8513d58373ab36b210d95b7f51e2c
-  data.tar.gz: d456194ef7e1c45a22a4419699e3f3cdcb82f09d987842ab15bcd3de009a2557cf1aa416da3d1ce0a8404fd3579d14103ca24e6b2495fa7684a421e9db6cf7b0
+  metadata.gz: e5cc4d39ac8e5f9d92edd240e836d5848f0b96798afbcab9c8116f8223142851d835b7bfd3e7a8d94e867951c4b995e0a66736a73b72d6a96f06fee6daf26bc9
+  data.tar.gz: 4f1facfbdffe1aca7fd0d10ff0c99d6f835b2633e94be49011b46127ca9cc7b76415930d5df0a961516000032b940f00e224c562923c06412c57f2896e50256f

data/.github/workflows/ruby.yml

@@ -21,10 +21,7 @@ jobs:
         uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{ matrix.ruby }}
-      - name: Install libsqlite3
-        run: |
-          sudo apt update -y && \
-          sudo apt install -y --no-install-recommends --no-install-suggests libsqlite3-dev
+          bundler-cache: true
       - name: Install dependencies
         run: bundle install --jobs 4 --retry 3
       - name: Run tests

data/.yardopts

@@ -1 +1 @@
---markup markdown --title 'Ronin FIXME Documentation' --protected
+--markup markdown --title 'Ronin::Web::Spider Documentation' --protected

data/ChangeLog.md

@@ -1,6 +1,9 @@
 ### 0.1.0 / 2023-XX-XX
 
+* Extracted and refactored from [ronin-web](https://github.com/ronin-rb/ronin-web/tree/v0.3.0.rc1).
+* Relicensed as LGPL-3.0.
 * Initial release:
+  * Requires `ruby` >= 3.0.0.
   * Built on top of the battle tested and versatile [spidr] gem.
   * Provides additional callback methods:
     * `every_host` - yields every unique host name that's spidered.

data/Gemfile

@@ -8,8 +8,8 @@ end
 
 # gem 'spidr',  '~> 0.7', github: 'postmodern/spidr'
 
-gem 'ronin-support',	       '~> 1.0', github: "ronin-rb/ronin-support",
-                                       branch: '1.0.0'
+# gem 'ronin-support',	       '~> 1.0', github: "ronin-rb/ronin-support",
+#                                        branch: 'main'
 
 group :development do
   gem 'rake'

data/README.md

@@ -2,6 +2,7 @@
 
 [![CI](https://github.com/ronin-rb/ronin-web-spider/actions/workflows/ruby.yml/badge.svg)](https://github.com/ronin-rb/ronin-web-spider/actions/workflows/ruby.yml)
 [![Code Climate](https://codeclimate.com/github/ronin-rb/ronin-web-spider.svg)](https://codeclimate.com/github/ronin-rb/ronin-web-spider)
+[![Gem Version](https://badge.fury.io/rb/ronin-web-spider.svg)](https://badge.fury.io/rb/ronin-web-spider)
 
 * [Website](https://ronin-rb.dev/)
 * [Source](https://github.com/ronin-rb/ronin-web-spider)
@@ -20,22 +21,35 @@ ronin-web-spider is a collection of common web spidering routines using the
 
 * Built on top of the battle tested and versatile [spidr] gem.
 * Provides additional callback methods:
-  * `every_host` - yields every unique host name that's spidered.
-  * `every_cert` - yields every unique SSL/TLS certificate encountered while
-    spidering.
-  * `every_favicon` - yields every favicon file that's encountered while
-    spidering.
-  * `every_html_comment` - yields every HTML comment.
-  * `every_javascript` - yields all JavaScript source code from either inline
-    `<script>` or `.js` files.
-  * `every_javascript_string` - yields every single-quoted or double-quoted
-    String literal from all JavaScript source code.
-  * `every_javascript_comment` - yields every JavaScript comment.
-  * `every_comment` - yields every HTML or JavaScript comment.
+  * [every_host][docs-every_host] - yields every unique host name that's
+    spidered.
+  * [every_cert][docs-every_cert] - yields every unique SSL/TLS certificate
+    encountered while spidering.
+  * [every_favicon][docs-every_favicon] - yields every favicon file that's
+    encountered while spidering.
+  * [every_html_comment][docs-every_html_comment] - yields every HTML comment.
+  * [every_javascript][docs-every_javascript] - yields all JavaScript source
+    code from either inline `<script>` or `.js` files.
+  * [every_javascript_string][docs-every_javascript_string] - yields every
+    single-quoted or double-quoted String literal from all JavaScript source
+    code.
+  * [every_javascript_comment][docs-every_javascript_comment] - yields every
+    JavaScript comment.
+  * [every_comment][docs-every_comment] - yields every HTML or JavaScript
+    comment.
 * Supports archiving spidered pages to a directory or git repository.
 * Has 94% documentation coverage.
 * Has 94% test coverage.
 
+[docs-every_host]: https://ronin-rb.dev/docs/ronin-web-spider/Ronin/Web/Spider/Agent.html#every_host-instance_method
+[docs-every_cert]: https://ronin-rb.dev/docs/ronin-web-spider/Ronin/Web/Spider/Agent.html#every_cert-instance_method
+[docs-every_favicon]: https://ronin-rb.dev/docs/ronin-web-spider/Ronin/Web/Spider/Agent.html#every_favicon-instance_method
+[docs-every_html_comment]: https://ronin-rb.dev/docs/ronin-web-spider/Ronin/Web/Spider/Agent.html#every_html_comment-instance_method
+[docs-every_javascript]: https://ronin-rb.dev/docs/ronin-web-spider/Ronin/Web/Spider/Agent.html#every_javascript-instance_method
+[docs-every_javascript_string]: https://ronin-rb.dev/docs/ronin-web-spider/Ronin/Web/Spider/Agent.html#every_javascript_string-instance_method
+[docs-every_javascript_comment]: https://ronin-rb.dev/docs/ronin-web-spider/Ronin/Web/Spider/Agent.html#every_javascript_comment-instance_method
+[docs-every_comment]: https://ronin-rb.dev/docs/ronin-web-spider/Ronin/Web/Spider/Agent.html#every_comment-instance_method
+
 ## Examples
 
 Spider a host:
@@ -43,41 +57,299 @@ Spider a host:
 ```ruby
 require 'ronin/web/spider'
 
-Ronin::Web::Spider.host('www.example.com') do |agent|
-  agent.ever_url do |url|
-    # ...
+Ronin::Web::Spider.start_at('http://tenderlovemaking.com/') do |agent|
+  # ...
+end
+```
+
+Spider a host:
+
+```ruby
+Ronin::Web::Spider.host('solnic.eu') do |agent|
+  # ...
+end
+```
+
+Spider a domain (and any sub-domains):
+
+```ruby
+Ronin::Web::Spider.domain('ruby-lang.org') do |agent|
+  # ...
+end
+```
+
+Spider a site:
+
+```ruby
+Ronin::Web::Spider.site('http://www.rubyflow.com/') do |agent|
+  # ...
+end
+```
+
+Spider multiple hosts:
+
+```ruby
+Ronin::Web::Spider.start_at('http://company.com/', hosts: ['company.com', /host[\d]+\.company\.com/]) do |agent|
+  # ...
+end
+```
+
+Do not spider certain links:
+
+```ruby
+Ronin::Web::Spider.site('http://company.com/', ignore_links: [%{^/blog/}]) do |agent|
+  # ...
+end
+```
+
+Do not spider links on certain ports:
+
+```ruby
+Ronin::Web::Spider.site('http://company.com/', ignore_ports: [8000, 8010, 8080]) do |agent|
+  # ...
+end
+```
+
+Do not spider links blacklisted in robots.txt:
+
+```ruby
+Ronin::Web::Spider.site('http://company.com/', robots: true) do |agent|
+  # ...
+end
+```
+
+Print out visited URLs:
+
+```ruby
+Ronin::Web::Spider.site('http://www.rubyinside.com/') do |spider|
+  spider.every_url { |url| puts url }
+end
+```
+
+Build a URL map of a site:
+
+```ruby
+url_map = Hash.new { |hash,key| hash[key] = [] }
+
+Ronin::Web::Spider.site('http://intranet.com/') do |spider|
+  spider.every_link do |origin,dest|
+    url_map[dest] << origin
+  end
+end
+```
+
+Print out the URLs that could not be requested:
+
+```ruby
+Ronin::Web::Spider.site('http://company.com/') do |spider|
+  spider.every_failed_url { |url| puts url }
+end
+```
+
+Finds all pages which have broken links:
+
+```ruby
+url_map = Hash.new { |hash,key| hash[key] = [] }
+
+spider = Ronin::Web::Spider.site('http://intranet.com/') do |spider|
+  spider.every_link do |origin,dest|
+    url_map[dest] << origin
+  end
+end
+
+spider.failures.each do |url|
+  puts "Broken link #{url} found in:"
+
+  url_map[url].each { |page| puts "  #{page}" }
+end
+```
+
+Search HTML and XML pages:
+
+```ruby
+Ronin::Web::Spider.site('http://company.com/') do |spider|
+  spider.every_page do |page|
+    puts ">>> #{page.url}"
+
+    page.search('//meta').each do |meta|
+      name = (meta.attributes['name'] || meta.attributes['http-equiv'])
+      value = meta.attributes['content']
+
+      puts "  #{name} = #{value}"
+    end
+  end
+end
+```
+
+Print out the titles from every page:
+
+```ruby
+Ronin::Web::Spider.site('https://www.ruby-lang.org/') do |spider|
+  spider.every_html_page do |page|
+    puts page.title
+  end
+end
+```
+
+Print out every HTTP redirect:
+
+```ruby
+Ronin::Web::Spider.host('company.com') do |spider|
+  spider.every_redirect_page do |page|
+    puts "#{page.url} -> #{page.headers['Location']}"
   end
+end
+```
+
+Find what kinds of web servers a host is using, by accessing the headers:
+
+```ruby
+servers = Set[]
 
-  agent.every_url_like(/.../) do |url|
-    # ...
+Ronin::Web::Spider.host('company.com') do |spider|
+  spider.all_headers do |headers|
+    servers << headers['server']
   end
+end
+```
 
-  agent.every_page do |page|
-    # ...
+Pause the spider on a forbidden page:
+
+```ruby
+Ronin::Web::Spider.host('company.com') do |spider|
+  spider.every_forbidden_page do |page|
+    spider.pause!
   end
 end
 ```
 
-See [Spidr::Agent] documentation for more agent methods.
+Skip the processing of a page:
 
-[Spidr::Agent]: https://rubydoc.info/gems/spidr/Spidr/Agent
+```ruby
+Ronin::Web::Spider.host('company.com') do |spider|
+  spider.every_missing_page do |page|
+    spider.skip_page!
+  end
+end
+```
 
-Spider a domain:
+Skip the processing of links:
 
 ```ruby
-Ronin::Web::Spider.domain('example.com') do |agent|
-  agent.every_page do |page|
-    # ...
+Ronin::Web::Spider.host('company.com') do |spider|
+  spider.every_url do |url|
+    if url.path.split('/').find { |dir| dir.to_i > 1000 }
+      spider.skip_link!
+    end
   end
 end
 ```
 
-Spider a website:
+Detect when a new host name is spidered:
 
 ```ruby
-Ronin::Web::Spider.site('https://www.example.com/index.html') do |agent|
-  agent.every_page do |page|
-    # ...
+Ronin::Web::Spider.domain('example.com') do |spider|
+  spider.every_host do |host|
+    puts "Spidering #{host} ..."
+  end
+end
+```
+
+Detect when a new SSL/TLS certificate is encountered:
+
+```ruby
+Ronin::Web::Spider.domain('example.com') do |spider|
+  spider.every_cert do |cert|
+    puts "Discovered new cert for #{cert.subject.command_name}, #{cert.subject_alt_name}"
+  end
+end
+```
+
+Print the MD5 checksum of every `favicon.ico` file:
+
+```ruby
+Ronin::Web::Spider.domain('example.com') do |spider|
+  spider.every_favicon do |page|
+    puts "#{page.url}: #{page.body.md5}"
+  end
+end
+```
+
+Print every HTML comment:
+
+```ruby
+Ronin::Web::Spider.domain('example.com') do |spider|
+  spider.every_html_comment do |comment|
+    puts comment
+  end
+end
+```
+
+Print all JavaScript source code:
+
+```ruby
+Ronin::Web::Spider.domain('example.com') do |spider|
+  spider.every_javascript do |js|
+    puts js
+  end
+end
+```
+
+Print every JavaScript string literal:
+
+```ruby
+Ronin::Web::Spider.domain('example.com') do |spider|
+  spider.every_javascript_string do |str|
+    puts str
+  end
+end
+```
+
+Print every JavaScript comment:
+
+```ruby
+Ronin::Web::Spider.domain('example.com') do |spider|
+  spider.every_javascript_comment do |comment|
+    puts comment
+  end
+end
+```
+
+Print every HTML and JavaScript comment:
+
+```ruby
+Ronin::Web::Spider.domain('example.com') do |spider|
+  spider.every_comment do |comment|
+    puts comment
+  end
+end
+```
+
+Spider a host and archive every web page:
+
+```ruby
+require 'ronin/web/spider'
+require 'ronin/web/spider/archive'
+
+Ronin::Web::Spider::Archive.open('path/to/root') do |archive|
+  Ronin::Web::Spider.every_page(host: 'example.com') do |page|
+    archive.write(page.url,page.body)
+  end
+end
+```
+
+Spider a host and archive every web page to a Git repository:
+
+```ruby
+require 'ronin/web/spider/git_archive'
+require 'ronin/web/spider'
+require 'date'
+
+Ronin::Web::Spider::GitArchive.open('path/to/root') do |archive|
+  archive.commit("Updated #{Date.today}") do
+    Ronin::Web::Spider.every_page(host: 'example.com') do |page|
+      archive.write(page.url,page.body)
+    end
   end
 end
 ```
@@ -119,7 +391,7 @@ gem.add_dependency 'ronin-web-spider', '~> 0.1'
 
 ## License
 
-Copyright (c) 2006-2022 Hal Brodigan (postmodern.mod3 at gmail.com)
+Copyright (c) 2006-2023 Hal Brodigan (postmodern.mod3 at gmail.com)
 
 ronin-web-spider is free software: you can redistribute it and/or modify
 it under the terms of the GNU Lesser General Public License as published

data/gemspec.yml

@@ -11,10 +11,10 @@ homepage: https://ronin-rb.dev/
 has_yard: true
 
 metadata:
-  documentation_uri: https://rubydoc.info/gems/ronin-web-spider
+  documentation_uri: https://ronin-rb.dev/docs/ronin-web-spider
   source_code_uri:   https://github.com/ronin-rb/ronin-web-spider
   bug_tracker_uri:   https://github.com/ronin-rb/ronin-web-spider/issues
-  changelog_uri:     https://github.com/ronin-rb/ronin-web-spider/blob/master/ChangeLog.md
+  changelog_uri:     https://github.com/ronin-rb/ronin-web-spider/blob/main/ChangeLog.md
   rubygems_mfa_required: 'true'
 
 required_ruby_version: ">= 3.0.0"

data/lib/ronin/web/spider/agent.rb

@@ -1,7 +1,7 @@
 #
 # ronin-web-spider - A collection of common web spidering routines.
 #
-# Copyright (c) 2006-2022 Hal Brodigan (postmodern.mod3 at gmail.com)
+# Copyright (c) 2006-2023 Hal Brodigan (postmodern.mod3 at gmail.com)
 #
 # ronin-web-spider is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Lesser General Public License as published
@@ -122,6 +122,8 @@ module Ronin
         # The visited host names.
         #
         # @return [Set<String>, nil]
+        #
+        # @api public
         attr_reader :visited_hosts
 
         #
@@ -132,6 +134,13 @@ module Ronin
         #
         # @yieldparam [String] host
         #
+        # @example
+        #   spider.every_host do |host|
+        #     puts "Spidring #{host} ..."
+        #   end
+        #
+        # @api public
+        #
         def every_host
           @visited_hosts ||= Set.new
 
@@ -147,6 +156,8 @@ module Ronin
         # All certificates encountered while spidering.
         #
         # @return [Array<Ronin::Support::Crypto::Cert>]
+        #
+        # @api public
         attr_reader :collected_certs
 
         #
@@ -157,6 +168,13 @@ module Ronin
         #
         # @yieldparam [Ronin::Support::Crypto::Cert]
         #
+        # @example
+        #   spider.every_cert do |cert|
+        #     puts "Discovered new cert for #{cert.subject.command_name}, #{cert.subject_alt_name}"
+        #   end
+        #
+        # @api public
+        #
         def every_cert
           @collected_certs ||= []
 
@@ -185,8 +203,15 @@ module Ronin
         # @yieldparam [Spidr::Page] favicon
         #   An encountered `.ico` file.
         #
+        # @example
+        #   spider.every_favicon do |page|
+        #     # ...
+        #   end
+        #
         # @see https://rubydoc.info/gems/spidr/Spidr/Page
         #
+        # @api public
+        #
         def every_favicon
           every_page do |page|
             yield page if page.icon?
@@ -197,12 +222,19 @@ module Ronin
         # Passes every non-empty HTML comment to the given block.
         #
         # @yield [comment]
-        #   The given block will be passevery HTML comment.
+        #   The given block will be pass every HTML comment.
         #
         # @yieldparam [String] comment
         #   The HTML comment inner text, with leading and trailing whitespace
         #   stripped.
         #
+        # @example
+        #   spider.every_html_comment do |comment|
+        #     puts comment
+        #   end
+        #
+        # @api public
+        #
         def every_html_comment
           every_html_page do |page|
             page.doc.xpath('//comment()').each do |comment|
@@ -224,6 +256,13 @@ module Ronin
         # @yieldparam [String] js
         #   The JavaScript source code.
         #
+        # @example
+        #   spider.every_javascript do |js|
+        #     puts js
+        #   end
+        #
+        # @api public
+        #
         def every_javascript
           # yield inner text of every `<script type="text/javascript">` tag
           # and every `.js` URL.
@@ -252,6 +291,13 @@ module Ronin
         # @yieldparam [String] string
         #   The parsed contents of a JavaScript string.
         #
+        # @example
+        #   spider.every_javascript_string do |str|
+        #    puts str
+        #  end
+        #
+        # @api public
+        #
         def every_javascript_string
           every_javascript do |js|
             js.scan(Support::Text::Patterns::STRING) do |js_string|
@@ -271,6 +317,13 @@ module Ronin
         # @yieldparam [String] comment
         #   The contents of a JavaScript comment.
         #
+        # @example
+        #   spider.every_javascript_comment do |comment|
+        #     puts comment
+        #   end
+        #
+        # @api public
+        #
         def every_javascript_comment(&block)
           every_javascript do |js|
             js.scan(Support::Text::Patterns::JAVASCRIPT_COMMENT,&block)
@@ -288,9 +341,16 @@ module Ronin
         # @yieldparam [String] comment
         #   The contents of a HTML or JavaScript comment.
         #
+        # @example
+        #   spider.every_comment do |comment|
+        #     puts comment
+        #   end
+        #
         # @see #every_html_comment
         # @see #every_javascript_comment
         #
+        # @api public
+        #
         def every_comment(&block)
           every_html_comment(&block)
           every_javascript_comment(&block)

data/lib/ronin/web/spider/archive.rb

@@ -29,6 +29,9 @@ module Ronin
       #
       # Spider a host and archive every web page:
       #
+      #     require 'ronin/web/spider'
+      #     require 'ronin/web/spider/archive'
+      #     
       #     Ronin::Web::Spider::Archive.open('path/to/root') do |archive|
       #       Ronin::Web::Spider.every_page(host: 'example.com') do |page|
       #         archive.write(page.url,page.body)

data/lib/ronin/web/spider/exceptions.rb

@@ -1,7 +1,7 @@
 #
 # ronin-web-spider - A collection of common web spidering routines.
 #
-# Copyright (c) 2006-2022 Hal Brodigan (postmodern.mod3 at gmail.com)
+# Copyright (c) 2006-2023 Hal Brodigan (postmodern.mod3 at gmail.com)
 #
 # ronin-web-spider is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Lesser General Public License as published

data/lib/ronin/web/spider/git_archive.rb

@@ -30,8 +30,8 @@ module Ronin
       #
       # Spider a host and archive every web page to a Git repository:
       #
-      #     require 'ronin/web/spider/git_archive'
       #     require 'ronin/web/spider'
+      #     require 'ronin/web/spider/git_archive'
       #     require 'date'
       #     
       #     Ronin::Web::Spider::GitArchive.open('path/to/root') do |archive|

data/lib/ronin/web/spider/version.rb

@@ -1,7 +1,7 @@
 #
 # ronin-web-spider - A collection of common web spidering routines.
 #
-# Copyright (c) 2006-2022 Hal Brodigan (postmodern.mod3 at gmail.com)
+# Copyright (c) 2006-2023 Hal Brodigan (postmodern.mod3 at gmail.com)
 #
 # ronin-web-spider is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Lesser General Public License as published
@@ -21,7 +21,7 @@ module Ronin
   module Web
     module Spider
       # ronin-web-spider version
-      VERSION = '0.1.0.beta1'
+      VERSION = '0.1.0'
     end
   end
 end