ronin-support-web 0.1.0.rc1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: '089304b3ff4267deef83cef4b882ad044686866a89602a2be076db9b15fdf95f'
+  data.tar.gz: af737ed89f5b6ccce135822eac932f0fd508a1b1dc3fff10952b72922c996015
+SHA512:
+  metadata.gz: 994d4405bf67505c514359632c1e1f2c158df16251e52baef280b3cfab84bad861e05625da02fb4e8d321c8e6f1eccc5430986ae06423194d7c5cec1c13b32a5
+  data.tar.gz: 9b25a6265a567da63eb784ac0f4a7b0f128b93f679afbffc13d24ab85a5391875a3732d30b9c91bf336327450e49aef4bf5733705bebf609cd13886e70f6d565

data/.document

@@ -0,0 +1,4 @@
+lib/**/*.rb
+- 
+ChangeLog.md
+COPYING.txt

data/.github/workflows/ruby.yml

@@ -0,0 +1,47 @@
+name: CI
+
+on: [ push, pull_request ]
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby:
+          - '3.0'
+          - '3.1'
+          - '3.2'
+          - '3.3'
+          - jruby
+          # - truffleruby
+    name: Ruby ${{ matrix.ruby }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+      # - name: Install dependencies
+      #   run: |
+      #     sudo apt update -y && \
+      #     sudo apt install -y --no-install-recommends --no-install-suggests FIXME
+      - name: Install dependencies
+        run: bundle install --jobs 4 --retry 3
+      - name: Run tests
+        run: bundle exec rake test
+
+  # rubocop linting
+  rubocop:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 3.0
+      - name: Install dependencies
+        run: bundle install --jobs 4 --retry 3
+      - name: Run rubocop
+        run: bundle exec rubocop --parallel

data/.gitignore

@@ -0,0 +1,13 @@
+/coverage
+/doc
+/pkg
+/man/*.[1-9]
+/vendor/bundle
+/Gemfile.lock
+/.bundle
+/.yardoc
+.DS_Store
+*.db
+*.log
+*.swp
+*~

data/.rspec

@@ -0,0 +1 @@
+--colour --format documentation

data/.rubocop.yml

@@ -0,0 +1,19 @@
+AllCops:
+  NewCops: enable
+  SuggestExtensions: false
+  TargetRubyVersion: 3.1
+
+inherit_gem:
+  rubocop-ronin: rubocop.yml
+
+#
+# ronin-support-web specific exceptions
+#
+
+# Need explicit whitespace for the JRuby's specific HTML heredocs.
+Layout/TrailingWhitespace:
+  Exclude:
+    - spec/agent/mixin_spec.rb
+    - spec/agent_spec.rb
+    - spec/html/mixin_spec.rb
+    - spec/html_spec.rb

data/.ruby-version

@@ -0,0 +1 @@
+ruby-3.1

data/.yardopts

@@ -0,0 +1 @@
+--markup markdown --title 'Ronin::Support::Web Documentation' --protected

data/COPYING.txt

@@ -0,0 +1,165 @@
+                   GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+
+  This version of the GNU Lesser General Public License incorporates
+the terms and conditions of version 3 of the GNU General Public
+License, supplemented by the additional permissions listed below.
+
+  0. Additional Definitions.
+
+  As used herein, "this License" refers to version 3 of the GNU Lesser
+General Public License, and the "GNU GPL" refers to version 3 of the GNU
+General Public License.
+
+  "The Library" refers to a covered work governed by this License,
+other than an Application or a Combined Work as defined below.
+
+  An "Application" is any work that makes use of an interface provided
+by the Library, but which is not otherwise based on the Library.
+Defining a subclass of a class defined by the Library is deemed a mode
+of using an interface provided by the Library.
+
+  A "Combined Work" is a work produced by combining or linking an
+Application with the Library.  The particular version of the Library
+with which the Combined Work was made is also called the "Linked
+Version".
+
+  The "Minimal Corresponding Source" for a Combined Work means the
+Corresponding Source for the Combined Work, excluding any source code
+for portions of the Combined Work that, considered in isolation, are
+based on the Application, and not on the Linked Version.
+
+  The "Corresponding Application Code" for a Combined Work means the
+object code and/or source code for the Application, including any data
+and utility programs needed for reproducing the Combined Work from the
+Application, but excluding the System Libraries of the Combined Work.
+
+  1. Exception to Section 3 of the GNU GPL.
+
+  You may convey a covered work under sections 3 and 4 of this License
+without being bound by section 3 of the GNU GPL.
+
+  2. Conveying Modified Versions.
+
+  If you modify a copy of the Library, and, in your modifications, a
+facility refers to a function or data to be supplied by an Application
+that uses the facility (other than as an argument passed when the
+facility is invoked), then you may convey a copy of the modified
+version:
+
+   a) under this License, provided that you make a good faith effort to
+   ensure that, in the event an Application does not supply the
+   function or data, the facility still operates, and performs
+   whatever part of its purpose remains meaningful, or
+
+   b) under the GNU GPL, with none of the additional permissions of
+   this License applicable to that copy.
+
+  3. Object Code Incorporating Material from Library Header Files.
+
+  The object code form of an Application may incorporate material from
+a header file that is part of the Library.  You may convey such object
+code under terms of your choice, provided that, if the incorporated
+material is not limited to numerical parameters, data structure
+layouts and accessors, or small macros, inline functions and templates
+(ten or fewer lines in length), you do both of the following:
+
+   a) Give prominent notice with each copy of the object code that the
+   Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the object code with a copy of the GNU GPL and this license
+   document.
+
+  4. Combined Works.
+
+  You may convey a Combined Work under terms of your choice that,
+taken together, effectively do not restrict modification of the
+portions of the Library contained in the Combined Work and reverse
+engineering for debugging such modifications, if you also do each of
+the following:
+
+   a) Give prominent notice with each copy of the Combined Work that
+   the Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the Combined Work with a copy of the GNU GPL and this license
+   document.
+
+   c) For a Combined Work that displays copyright notices during
+   execution, include the copyright notice for the Library among
+   these notices, as well as a reference directing the user to the
+   copies of the GNU GPL and this license document.
+
+   d) Do one of the following:
+
+       0) Convey the Minimal Corresponding Source under the terms of this
+       License, and the Corresponding Application Code in a form
+       suitable for, and under terms that permit, the user to
+       recombine or relink the Application with a modified version of
+       the Linked Version to produce a modified Combined Work, in the
+       manner specified by section 6 of the GNU GPL for conveying
+       Corresponding Source.
+
+       1) Use a suitable shared library mechanism for linking with the
+       Library.  A suitable mechanism is one that (a) uses at run time
+       a copy of the Library already present on the user's computer
+       system, and (b) will operate properly with a modified version
+       of the Library that is interface-compatible with the Linked
+       Version.
+
+   e) Provide Installation Information, but only if you would otherwise
+   be required to provide such information under section 6 of the
+   GNU GPL, and only to the extent that such information is
+   necessary to install and execute a modified version of the
+   Combined Work produced by recombining or relinking the
+   Application with a modified version of the Linked Version. (If
+   you use option 4d0, the Installation Information must accompany
+   the Minimal Corresponding Source and Corresponding Application
+   Code. If you use option 4d1, you must provide the Installation
+   Information in the manner specified by section 6 of the GNU GPL
+   for conveying Corresponding Source.)
+
+  5. Combined Libraries.
+
+  You may place library facilities that are a work based on the
+Library side by side in a single library together with other library
+facilities that are not Applications and are not covered by this
+License, and convey such a combined library under terms of your
+choice, if you do both of the following:
+
+   a) Accompany the combined library with a copy of the same work based
+   on the Library, uncombined with any other library facilities,
+   conveyed under the terms of this License.
+
+   b) Give prominent notice with the combined library that part of it
+   is a work based on the Library, and explaining where to find the
+   accompanying uncombined form of the same work.
+
+  6. Revised Versions of the GNU Lesser General Public License.
+
+  The Free Software Foundation may publish revised and/or new versions
+of the GNU Lesser General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+  Each version is given a distinguishing version number. If the
+Library as you received it specifies that a certain numbered version
+of the GNU Lesser General Public License "or any later version"
+applies to it, you have the option of following the terms and
+conditions either of that published version or of any later version
+published by the Free Software Foundation. If the Library as you
+received it does not specify a version number of the GNU Lesser
+General Public License, you may choose any version of the GNU Lesser
+General Public License ever published by the Free Software Foundation.
+
+  If the Library as you received it specifies that a proxy can decide
+whether future versions of the GNU Lesser General Public License shall
+apply, that proxy's public statement of acceptance of any version is
+permanent authorization for you to choose that version for the
+Library.

data/ChangeLog.md

@@ -0,0 +1,10 @@
+### 0.1.0 / 2024-XX-XX
+
+* Initial release:
+  * Provides helper methods for parsing HTML/XML.
+    * Also provides additional extensions to [Nokogiri][nokogiri] using
+      [nokogiri-ext].
+  * Provides helper methods for working with WebSockets.
+
+[nokogiri]: https://nokogiri.org/
+[nokogiri-ext]: https://github.com/postmodern/nokogiri-ext#readme

data/Gemfile

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+gemspec
+
+platform :jruby do
+  gem 'jruby-openssl', '~> 0.7'
+end
+
+# gem 'nokogiri-ext', '~> 0.1', github: 'postmodern/nokogiri-ext',
+#                               branch: 'main'
+
+# Ronin dependencies
+# gem 'ronin-support', '~> 1.1', github: 'ronin-rb/ronin-support',
+#                                branch: 'main'
+
+group :development do
+  gem 'rake'
+  gem 'rubygems-tasks',   '~> 0.2'
+
+  gem 'rspec',            '~> 3.0'
+  gem 'webmock',          '~> 3.0'
+  gem 'simplecov',        '~> 0.20'
+
+  gem 'kramdown',         '~> 2.0'
+  gem 'redcarpet',        platform: :mri
+  gem 'yard',             '~> 0.9'
+  gem 'yard-spellcheck',  require: false
+
+  gem 'dead_end',         require: false
+  gem 'sord',             require: false, platform: :mri
+  gem 'stackprof',        require: false, platform: :mri
+  gem 'rubocop',          require: false, platform: :mri
+  gem 'rubocop-ronin',    require: false, platform: :mri
+end

data/README.md

@@ -0,0 +1,321 @@
+# ronin-support-web
+
+[![CI](https://github.com/ronin-rb/ronin-support-web/actions/workflows/ruby.yml/badge.svg)](https://github.com/ronin-rb/ronin-support-web/actions/workflows/ruby.yml)
+[![Code Climate](https://codeclimate.com/github/ronin-rb/ronin-support-web.svg)](https://codeclimate.com/github/ronin-rb/ronin-support-web)
+
+* [Website](https://ronin-rb.dev/)
+* [Source](https://github.com/ronin-rb/ronin-support-web)
+* [Issues](https://github.com/ronin-rb/ronin-support-web/issues)
+* [Documentation](https://ronin-rb.dev/docs/ronin-support-web)
+* [Discord](https://discord.gg/6WAb3PsVX9) |
+  [Mastodon](https://infosec.exchange/@ronin_rb)
+
+## Description
+
+ronin-support-web is a web support library for ronin-rb. ronin-support-web
+provides many helper methods for parsing HTML/XML, fetching web pages, etc.
+
+## Features
+
+* Provides helper methods for parsing HTML/XML.
+  * Also provides additional extensions to [Nokogiri][nokogiri] using
+    [nokogiri-ext].
+* Provides helper methods for working with WebSockets.
+
+## Examples
+
+```ruby
+require 'ronin/support/web'
+include Ronin::Support::Web
+
+html_parse "<html>...</html>"
+# => #<Nokogiri::HTML::Document:...>
+```
+
+### HTML
+
+Parse an HTML string:
+
+```ruby
+doc = html_parse("<html>\n  <body>\n    <p>Hello world</p>\n  </body>\n</html>\n")
+# => 
+# #(Document:0x6ab8 {
+#   name = "document",
+#   children = [
+#     #(DTD:0x6be4 { name = "html" }),
+#     #(Element:0x6cd4 {
+#       name = "html",
+#       children = [
+#         #(Text "\n  "),
+#         #(Element:0x6e64 {
+#           name = "body",
+#           children = [
+#             #(Text "\n    "),
+#             #(Element:0x6ff4 { name = "p", children = [ #(Text "Hello world")] }),
+#             #(Text "\n  ")]
+#           }),
+#         #(Text "\n")]
+#       })]
+#   })
+```
+
+Parse a HTML file:
+
+```ruby
+doc = html_open("index.html")
+# => #<Nokogiri::HTML::Document:...>
+```
+
+Searching an HTML document using [XPath] or CSS-path:
+
+```ruby
+nodes = doc.search('//div/p')
+nodes = doc.search('div p.class')
+# => [#<Nokogiri::HTML::Element:...>, ...]
+
+node = doc.at('#id')
+# => #<Nokogiri::HTML::Element:...>
+```
+
+Build a HTML document:
+
+```ruby
+doc = html_build do
+  html {
+    head {
+      script(type: 'text/javascript', src: 'redirect.js')
+    }
+  }
+end
+
+puts doc.to_html
+# <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" "http://www.w3.org/TR/REC-html40/loose.dtd">
+# <html><head><script src="redirect.js" type="text/javascript"></script></head></html>
+```
+
+### XML
+
+Parse an XML response body:
+
+```ruby
+xml_parse("<?xml version=\"1.0\"?>\n<users>\n  <user>\n    <name>admin</name>\n    <password>0mni</password>\n  <user>\n</users>\n")
+# =>
+# #(Document:0xdebc {
+#   name = "document",
+#   children = [
+#     #(Element:0xdfe8 {
+#       name = "users",
+#       children = [
+#         #(Text "\n  "),
+#         #(Element:0xe178 {
+#           name = "user",
+#           children = [
+#             #(Text "\n    "),
+#             #(Element:0xe308 { name = "name", children = [ #(Text "admin")] }),
+#             #(Text "\n    "),
+#             #(Element:0xe538 { name = "password", children = [ #(Text "0mni")] }),
+#             #(Text "\n  "),
+#             #(Element:0xe768 { name = "user", children = [ #(Text "\n")] }),
+#             #(Text "\n")]
+#           })]
+#       })]
+#   })
+```
+
+Parse a XML file:
+
+```ruby
+doc = html_open("data.xml")
+# => #<Nokogiri:XML:::Document:...>
+```
+
+Searching an XML document using [XPath]:
+
+```ruby
+users = doc.search('//user')
+# => [#<Nokogiri::XML::Element:...>, ...]
+
+admin = doc.at('//user[@name="admin"]')
+# => #<Nokogiri::XML::Element:...>
+```
+
+Build a XML document:
+
+```ruby
+doc = xml_build do
+  playlist {
+    mp3 {
+      file { text('02 THE WAIT.mp3') }
+      artist { text('Evil Nine') }
+      track { text('The Wait feat David Autokratz') }
+      duration { text('1000000000') }
+    }
+  }
+end
+
+puts doc.to_xml
+# <?xml version="1.0"?>
+# <playlist>
+#   <mp3>
+#     <file>02 THE WAIT.mp3</file>
+#     <artist>Evil Nine</artist>
+#     <track>The Wait feat David Autokratz</track>
+#     <duration>1000000000</duration>
+#   </mp3>
+# </playlist>
+```
+
+### Web Requests
+
+Gets a URL and follows any redirects:
+
+```ruby
+get 'https://example.com/'
+# => #<Net::HTTPResponse:...>
+```
+
+Gets a URL and parses the HTML response:
+
+```ruby
+get_html 'https://example.com/'
+# => #<Nokogiri::HTML::Document:...>
+```
+
+Gets a URL and parses the XML response:
+
+```ruby
+get_xml 'https://example.com/sitemap.xml'
+# => #<Nokogiri::XML::Document:...>
+```
+
+Gets a URL and parses the JSON response:
+
+```ruby
+get_json 'https://example.com/api/endpoint.json'
+# => {...}
+```
+
+POSTs to a URL and follows any redirects:
+
+```ruby
+post 'https://example.com/form', form_data: {'foo' => 'bar'}
+# => #<Net::HTTPResponse:...>
+```
+
+POSTs to a URL and parses the HTML response:
+
+```ruby
+post_html 'https://example.com/form', form_data: {'foo' => 'bar'}
+# => #<Nokogiri::HTML::Document:...>
+```
+
+POSTs to a URL and parses the XML response:
+
+```ruby
+post_xml 'https://example.com/form', form_data: {'foo' => 'bar'}
+# => #<Nokogiri::XML::Document:...>
+```
+
+POSTs to a URL and parses the JSON response:
+
+```ruby
+post_json 'https://example.com/api/endpoint.json', json: {foo: 'bar'}
+# => {...}
+```
+
+### WebSockets
+
+Connecting to a WebSocket:
+
+```ruby
+websocket = websocket_connect('ws://websocket-echo.com')
+
+websocket.send_frame("foo bar")
+# => 13
+websocket.recv_frame
+# => <WebSocket::Frame::Incoming::Client:0x000227a4 @decoded=true, @code=nil, @data="foo bar", @version=13, @handler=#<WebSocket::Frame::Handler::Handler07:0x00007f7d4c9b0ed0 @frame=<WebSocket::Frame::Incoming::Client:0x000227a4 @decoded=true, @code=nil, @data="foo bar", @version=13, @handler=#<WebSocket::Frame::Handler::Handler07:0x00007f7d4c9b0ed0 ...>, @type=:text>, @application_data_buffer=nil>, @type=:text>
+websocket.send_frame("hello world")
+# => 17
+websocket.recv
+# => "hello world"
+```
+
+Starting a WebSocket server and receiving connections:
+
+```ruby
+server = websocket_server('ws://localhost:1337/')
+client = server.accept
+client.send("hello")
+client.recv
+# => "good, how are you"
+```
+```ruby
+client = websocket_client('ws://localhost:1337/')
+client.recv
+# => "hello"
+client.send("good, how are you")
+```
+
+## Requirements
+
+* [Ruby] >= 3.0.0
+* [ronin-support] ~> 1.1
+* [nokogiri] ~> 1.4
+* [nokogiri-ext] ~> 0.1
+* [websocket] ~> 1.2
+
+## Install
+
+```shell
+$ gem install ronin-support-web
+```
+
+### Gemfile
+
+```ruby
+gem 'ronin-support-web', '~> 0.1'
+```
+
+### gemspec
+
+```ruby
+gem.add_dependency 'ronin-support-web', '~> 0.1'
+```
+
+## Development
+
+1. [Fork It!](https://github.com/ronin-rb/ronin-support-web/fork)
+2. Clone It!
+3. `cd ronin-support-web/`
+4. `bundle install`
+5. `git checkout -b my_feature`
+6. Code It!
+7. `bundle exec rake spec`
+8. `git push origin my_feature`
+
+## License
+
+ronin-support-web - A web support library for ronin-rb.
+
+Copyright (c) 2023-2024 Hal Brodigan (postmodern.mod3@gmail.com)
+
+ronin-support-web is free software: you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published
+by the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+ronin-support-web is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License
+along with ronin-support-web.  If not, see <https://www.gnu.org/licenses/>.
+
+[Ruby]: https://www.ruby-lang.org
+[nokogiri]: https://nokogiri.org/
+[nokogiri-ext]: https://github.com/postmodern/nokogiri-ext#readme
+[ronin-support]: https://github.com/ronin-rb/ronin-support#readme
+[websocket]: https://github.com/imanel/websocket-ruby#readme
+
+[XPath]: https://developer.mozilla.org/en-US/docs/Web/XPath

data/Rakefile

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+begin
+  require 'bundler'
+rescue LoadError => e
+  warn e.message
+  warn "Run `gem install bundler` to install Bundler"
+  exit(-1)
+end
+
+begin
+  Bundler.setup(:development)
+rescue Bundler::BundlerError => e
+  warn e.message
+  warn "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+
+require 'rake'
+
+require 'rubygems/tasks'
+Gem::Tasks.new(sign: {checksum: true, pgp: true})
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new
+task :test    => :spec
+task :default => :spec
+
+require 'yard'
+YARD::Rake::YardocTask.new
+task :docs => :yard