ronin-web-spider 0.1.0.beta1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: f4a08a150f4301148cc86877702ea981f2083d173c3101ee6047d44cfc5f076c
+  data.tar.gz: 8fb751613e0cee2161d08b338cce434b252ba83f62971275b5a33d0060cec567
+SHA512:
+  metadata.gz: 5af53f0fbf55f13d8ccd2ff9fd13d7cfba7d6ce7c8aae90e5b44a467617212eb77247f9e343da118bdc8a369a9cbb7d240f8513d58373ab36b210d95b7f51e2c
+  data.tar.gz: d456194ef7e1c45a22a4419699e3f3cdcb82f09d987842ab15bcd3de009a2557cf1aa416da3d1ce0a8404fd3579d14103ca24e6b2495fa7684a421e9db6cf7b0

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+- 
+ChangeLog.md
+COPYING.txt
+man/*.md

data/.github/workflows/ruby.yml

@@ -0,0 +1,31 @@
+name: CI
+
+on: [ push, pull_request ]
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby:
+          - '3.0'
+          - '3.1'
+          - '3.2'
+          - jruby
+          - truffleruby
+    name: Ruby ${{ matrix.ruby }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+      - name: Install libsqlite3
+        run: |
+          sudo apt update -y && \
+          sudo apt install -y --no-install-recommends --no-install-suggests libsqlite3-dev
+      - name: Install dependencies
+        run: bundle install --jobs 4 --retry 3
+      - name: Run tests
+        run: bundle exec rake test

data/.gitignore

@@ -0,0 +1,13 @@
+/coverage
+/doc
+/pkg
+/man/*.[1-9]
+/vendor/bundle
+/Gemfile.lock
+/.bundle
+/.yardoc
+.DS_Store
+*.db
+*.log
+*.swp
+*~

data/.rspec

@@ -0,0 +1 @@
+--colour --format documentation

data/.ruby-version

@@ -0,0 +1 @@
+ruby-3.1

data/.yardopts

@@ -0,0 +1 @@
+--markup markdown --title 'Ronin FIXME Documentation' --protected

data/COPYING.txt

@@ -0,0 +1,165 @@
+                   GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+
+  This version of the GNU Lesser General Public License incorporates
+the terms and conditions of version 3 of the GNU General Public
+License, supplemented by the additional permissions listed below.
+
+  0. Additional Definitions.
+
+  As used herein, "this License" refers to version 3 of the GNU Lesser
+General Public License, and the "GNU GPL" refers to version 3 of the GNU
+General Public License.
+
+  "The Library" refers to a covered work governed by this License,
+other than an Application or a Combined Work as defined below.
+
+  An "Application" is any work that makes use of an interface provided
+by the Library, but which is not otherwise based on the Library.
+Defining a subclass of a class defined by the Library is deemed a mode
+of using an interface provided by the Library.
+
+  A "Combined Work" is a work produced by combining or linking an
+Application with the Library.  The particular version of the Library
+with which the Combined Work was made is also called the "Linked
+Version".
+
+  The "Minimal Corresponding Source" for a Combined Work means the
+Corresponding Source for the Combined Work, excluding any source code
+for portions of the Combined Work that, considered in isolation, are
+based on the Application, and not on the Linked Version.
+
+  The "Corresponding Application Code" for a Combined Work means the
+object code and/or source code for the Application, including any data
+and utility programs needed for reproducing the Combined Work from the
+Application, but excluding the System Libraries of the Combined Work.
+
+  1. Exception to Section 3 of the GNU GPL.
+
+  You may convey a covered work under sections 3 and 4 of this License
+without being bound by section 3 of the GNU GPL.
+
+  2. Conveying Modified Versions.
+
+  If you modify a copy of the Library, and, in your modifications, a
+facility refers to a function or data to be supplied by an Application
+that uses the facility (other than as an argument passed when the
+facility is invoked), then you may convey a copy of the modified
+version:
+
+   a) under this License, provided that you make a good faith effort to
+   ensure that, in the event an Application does not supply the
+   function or data, the facility still operates, and performs
+   whatever part of its purpose remains meaningful, or
+
+   b) under the GNU GPL, with none of the additional permissions of
+   this License applicable to that copy.
+
+  3. Object Code Incorporating Material from Library Header Files.
+
+  The object code form of an Application may incorporate material from
+a header file that is part of the Library.  You may convey such object
+code under terms of your choice, provided that, if the incorporated
+material is not limited to numerical parameters, data structure
+layouts and accessors, or small macros, inline functions and templates
+(ten or fewer lines in length), you do both of the following:
+
+   a) Give prominent notice with each copy of the object code that the
+   Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the object code with a copy of the GNU GPL and this license
+   document.
+
+  4. Combined Works.
+
+  You may convey a Combined Work under terms of your choice that,
+taken together, effectively do not restrict modification of the
+portions of the Library contained in the Combined Work and reverse
+engineering for debugging such modifications, if you also do each of
+the following:
+
+   a) Give prominent notice with each copy of the Combined Work that
+   the Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the Combined Work with a copy of the GNU GPL and this license
+   document.
+
+   c) For a Combined Work that displays copyright notices during
+   execution, include the copyright notice for the Library among
+   these notices, as well as a reference directing the user to the
+   copies of the GNU GPL and this license document.
+
+   d) Do one of the following:
+
+       0) Convey the Minimal Corresponding Source under the terms of this
+       License, and the Corresponding Application Code in a form
+       suitable for, and under terms that permit, the user to
+       recombine or relink the Application with a modified version of
+       the Linked Version to produce a modified Combined Work, in the
+       manner specified by section 6 of the GNU GPL for conveying
+       Corresponding Source.
+
+       1) Use a suitable shared library mechanism for linking with the
+       Library.  A suitable mechanism is one that (a) uses at run time
+       a copy of the Library already present on the user's computer
+       system, and (b) will operate properly with a modified version
+       of the Library that is interface-compatible with the Linked
+       Version.
+
+   e) Provide Installation Information, but only if you would otherwise
+   be required to provide such information under section 6 of the
+   GNU GPL, and only to the extent that such information is
+   necessary to install and execute a modified version of the
+   Combined Work produced by recombining or relinking the
+   Application with a modified version of the Linked Version. (If
+   you use option 4d0, the Installation Information must accompany
+   the Minimal Corresponding Source and Corresponding Application
+   Code. If you use option 4d1, you must provide the Installation
+   Information in the manner specified by section 6 of the GNU GPL
+   for conveying Corresponding Source.)
+
+  5. Combined Libraries.
+
+  You may place library facilities that are a work based on the
+Library side by side in a single library together with other library
+facilities that are not Applications and are not covered by this
+License, and convey such a combined library under terms of your
+choice, if you do both of the following:
+
+   a) Accompany the combined library with a copy of the same work based
+   on the Library, uncombined with any other library facilities,
+   conveyed under the terms of this License.
+
+   b) Give prominent notice with the combined library that part of it
+   is a work based on the Library, and explaining where to find the
+   accompanying uncombined form of the same work.
+
+  6. Revised Versions of the GNU Lesser General Public License.
+
+  The Free Software Foundation may publish revised and/or new versions
+of the GNU Lesser General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+  Each version is given a distinguishing version number. If the
+Library as you received it specifies that a certain numbered version
+of the GNU Lesser General Public License "or any later version"
+applies to it, you have the option of following the terms and
+conditions either of that published version or of any later version
+published by the Free Software Foundation. If the Library as you
+received it does not specify a version number of the GNU Lesser
+General Public License, you may choose any version of the GNU Lesser
+General Public License ever published by the Free Software Foundation.
+
+  If the Library as you received it specifies that a proxy can decide
+whether future versions of the GNU Lesser General Public License shall
+apply, that proxy's public statement of acceptance of any version is
+permanent authorization for you to choose that version for the
+Library.

data/ChangeLog.md

@@ -0,0 +1,19 @@
+### 0.1.0 / 2023-XX-XX
+
+* Initial release:
+  * Built on top of the battle tested and versatile [spidr] gem.
+  * Provides additional callback methods:
+    * `every_host` - yields every unique host name that's spidered.
+    * `every_cert` - yields every unique SSL/TLS certificate encountered while
+      spidering.
+    * `every_favicon` - yields every favicon file that's encountered while
+      spidering.
+    * `every_html_comment` - yields every HTML comment.
+    * `every_javascript` - yields all JavaScript source code from either inline
+      `<script>` or `.js` files.
+    * `every_javascript_string` - yields every single-quoted or double-quoted
+      String literal from all JavaScript source code.
+    * `every_javascript_comment` - yields every JavaScript comment.
+    * `every_comment` - yields every HTML or JavaScript comment.
+  * Supports archiving spidered pages to a directory or git repository.
+

data/Gemfile

@@ -0,0 +1,31 @@
+source 'https://rubygems.org'
+
+gemspec
+
+platform :jruby do
+  gem 'jruby-openssl',	'~> 0.7'
+end
+
+# gem 'spidr',  '~> 0.7', github: 'postmodern/spidr'
+
+gem 'ronin-support',	       '~> 1.0', github: "ronin-rb/ronin-support",
+                                       branch: '1.0.0'
+
+group :development do
+  gem 'rake'
+  gem 'rubygems-tasks', '~> 0.2'
+
+  gem 'rspec',           '~> 3.0'
+  gem 'webmock',         '~> 2.0'
+  gem 'sinatra',         '~> 1.0'
+  gem 'simplecov',       '~> 0.20'
+
+  gem 'kramdown',        '~> 2.0'
+  gem 'redcarpet',       platform: :mri
+  gem 'yard',            '~> 0.9'
+  gem 'yard-spellcheck', require: false
+
+  gem 'dead_end',        require: false
+  gem 'sord',            require: false, platform: :mri
+  gem 'stackprof',       require: false, platform: :mri
+end

data/README.md

@@ -0,0 +1,139 @@
+# ronin-web-spider
+
+[![CI](https://github.com/ronin-rb/ronin-web-spider/actions/workflows/ruby.yml/badge.svg)](https://github.com/ronin-rb/ronin-web-spider/actions/workflows/ruby.yml)
+[![Code Climate](https://codeclimate.com/github/ronin-rb/ronin-web-spider.svg)](https://codeclimate.com/github/ronin-rb/ronin-web-spider)
+
+* [Website](https://ronin-rb.dev/)
+* [Source](https://github.com/ronin-rb/ronin-web-spider)
+* [Issues](https://github.com/ronin-rb/ronin-web-spider/issues)
+* [Documentation](https://ronin-rb.dev/docs/ronin-web-spider/frames)
+* [Discord](https://discord.gg/6WAb3PsVX9) |
+  [Twitter](https://twitter.com/ronin_rb) |
+  [Mastodon](https://infosec.exchange/@ronin_rb)
+
+## Description
+
+ronin-web-spider is a collection of common web spidering routines using the
+[spidr] gem.
+
+## Features
+
+* Built on top of the battle tested and versatile [spidr] gem.
+* Provides additional callback methods:
+  * `every_host` - yields every unique host name that's spidered.
+  * `every_cert` - yields every unique SSL/TLS certificate encountered while
+    spidering.
+  * `every_favicon` - yields every favicon file that's encountered while
+    spidering.
+  * `every_html_comment` - yields every HTML comment.
+  * `every_javascript` - yields all JavaScript source code from either inline
+    `<script>` or `.js` files.
+  * `every_javascript_string` - yields every single-quoted or double-quoted
+    String literal from all JavaScript source code.
+  * `every_javascript_comment` - yields every JavaScript comment.
+  * `every_comment` - yields every HTML or JavaScript comment.
+* Supports archiving spidered pages to a directory or git repository.
+* Has 94% documentation coverage.
+* Has 94% test coverage.
+
+## Examples
+
+Spider a host:
+
+```ruby
+require 'ronin/web/spider'
+
+Ronin::Web::Spider.host('www.example.com') do |agent|
+  agent.ever_url do |url|
+    # ...
+  end
+
+  agent.every_url_like(/.../) do |url|
+    # ...
+  end
+
+  agent.every_page do |page|
+    # ...
+  end
+end
+```
+
+See [Spidr::Agent] documentation for more agent methods.
+
+[Spidr::Agent]: https://rubydoc.info/gems/spidr/Spidr/Agent
+
+Spider a domain:
+
+```ruby
+Ronin::Web::Spider.domain('example.com') do |agent|
+  agent.every_page do |page|
+    # ...
+  end
+end
+```
+
+Spider a website:
+
+```ruby
+Ronin::Web::Spider.site('https://www.example.com/index.html') do |agent|
+  agent.every_page do |page|
+    # ...
+  end
+end
+```
+
+## Requirements
+
+* [Ruby] >= 3.0.0
+* [spidr] ~> 0.7
+* [ronin-support] ~> 1.0
+
+## Install
+
+```shell
+$ gem install ronin-web-spider
+```
+
+### Gemfile
+
+```ruby
+gem 'ronin-web-spider', '~> 0.1'
+```
+
+### gemspec
+
+```ruby
+gem.add_dependency 'ronin-web-spider', '~> 0.1'
+```
+
+## Development
+
+1. [Fork It!](https://github.com/ronin-rb/ronin-web-spider/fork)
+2. Clone It!
+3. `cd ronin-web-spider/`
+4. `bundle install`
+5. `git checkout -b my_feature`
+6. Code It!
+7. `bundle exec rake spec`
+8. `git push origin my_feature`
+
+## License
+
+Copyright (c) 2006-2022 Hal Brodigan (postmodern.mod3 at gmail.com)
+
+ronin-web-spider is free software: you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published
+by the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+ronin-web-spider is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License
+along with ronin-web-spider.  If not, see <https://www.gnu.org/licenses/>.
+
+[Ruby]: https://www.ruby-lang.org
+[spidr]: https://github.com/postmodern/spidr#readme
+[ronin-support]: https://github.com/ronin-rb/ronin-support#readme

data/Rakefile

@@ -0,0 +1,31 @@
+require 'rubygems'
+
+begin
+  require 'bundler'
+rescue LoadError => e
+  warn e.message
+  warn "Run `gem install bundler` to install Bundler"
+  exit -1
+end
+
+begin
+  Bundler.setup(:development)
+rescue Bundler::BundlerError => e
+  warn e.message
+  warn "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+
+require 'rake'
+
+require 'rubygems/tasks'
+Gem::Tasks.new(sign: {checksum: true, pgp: true})
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new
+task :test    => :spec
+task :default => :spec
+
+require 'yard'
+YARD::Rake::YardocTask.new
+task :docs => :yard

data/gemspec.yml

@@ -0,0 +1,27 @@
+name: ronin-web-spider
+summary: collection of common web spidering routines
+description:
+  ronin-web-spider is a collection of common web spidering routines using the
+  spidr gem.
+
+license: LGPL-3.0
+authors: Postmodern
+email: postmodern.mod3@gmail.com
+homepage: https://ronin-rb.dev/
+has_yard: true
+
+metadata:
+  documentation_uri: https://rubydoc.info/gems/ronin-web-spider
+  source_code_uri:   https://github.com/ronin-rb/ronin-web-spider
+  bug_tracker_uri:   https://github.com/ronin-rb/ronin-web-spider/issues
+  changelog_uri:     https://github.com/ronin-rb/ronin-web-spider/blob/master/ChangeLog.md
+  rubygems_mfa_required: 'true'
+
+required_ruby_version: ">= 3.0.0"
+
+dependencies:
+  spidr: ~> 0.7
+  ronin-support: ~> 1.0
+
+development_dependencies:
+  bundler: ~> 2.0