ronin-web 1.0.0.beta3 → 1.0.0.beta4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4e36e87f767a665ff3dafb90022b8cb0b0d8d7d9ebc23255c873dcebde04d7b1
-  data.tar.gz: af5f942251ddefcfd8940d2de5cc688edbd6e55ba96715d8798b0e85efff33f6
+  metadata.gz: 37043413378124ed9793dc5393fe121be29e770f55e0cf93db3bbf91b59927d0
+  data.tar.gz: 5a77f64778e370b2d8f8c0f2df6838aa62719551a2021c6a0bbd5a9a4ba15ef1
 SHA512:
-  metadata.gz: 9e0bc704b1177e0f958c00a0f1f33d606258c78186dc2ac38de85d7188d29e9911aab5d337cb39a7e90798865841b947abd8c6e928325df2a394e7a61b6576a2
-  data.tar.gz: c7dd821942055424f990ab2f5511252308d9b12bf3bdb2ccb07281d523d40066425989d1645a27f464321147973b1ff12410c6bcc79cece614d09efbaf730e57
+  metadata.gz: b03fec155874e2c399db5991d03089a15c1c2193c3af3ba7c501c1c26ccd48c8fc6baaa650cda1c53508e43c5bbaed757cc7223279d494e635553f369202777e
+  data.tar.gz: 1a163daa3263ef36c36baa1d867a7add8678742743fb5a830ba10cbfbfcb05bd9c18b97d186538220f599298bc81c3a0b7861dabb4f9a69c1813ca34b0701936

data/.github/workflows/ruby.yml

@@ -21,6 +21,7 @@ jobs:
         uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
       - name: Install dependencies
         run: bundle install --jobs 4 --retry 3
       - name: Run tests

data/README.md

@@ -2,6 +2,7 @@
 
 [![CI](https://github.com/ronin-rb/ronin-web/actions/workflows/ruby.yml/badge.svg)](https://github.com/ronin-rb/ronin-web/actions/workflows/ruby.yml)
 [![Code Climate](https://codeclimate.com/github/ronin-rb/ronin-web.svg)](https://codeclimate.com/github/ronin-rb/ronin-web)
+[![Gem Version](https://badge.fury.io/rb/ronin-web.svg)](https://badge.fury.io/rb/ronin-web)
 
 * [Source](https://github.com/ronin-rb/ronin-web)
 * [Issues](https://github.com/ronin-rb/ronin-web/issues)

data/lib/ronin/web/cli/commands/spider.rb

@@ -83,6 +83,7 @@ module Ronin
         #         --print-status               Print the status codes for each URL
         #         --print-headers              Print response headers for each URL
         #         --print-header NAME          Prints a specific header
+        #         --history FILE               The history file
         #         --archive DIR                Archive every visited page to the DIR
         #         --git-archive DIR            Archive every visited page to the git repository
         #     -X, --xpath XPATH                Evaluates the XPath on each HTML page
@@ -390,7 +391,7 @@ module Ronin
                         },
                         desc: 'Spiders the website, starting at the URL'
 
-          option :print_verbose, desc: 'Print the status codes for each URL'
+          option :print_stauts, desc: 'Print the status codes for each URL'
 
           option :print_headers, desc: 'Print response headers for each URL'
 
@@ -400,6 +401,12 @@ module Ronin
                                 },
                                 desc: 'Prints a specific header'
 
+          option :history, value: {
+                             type: String,
+                             usage: 'FILE'
+                           },
+                           desc: 'The history file'
+
           option :archive, value: {
                              type: String,
                              usage: 'DIR'
@@ -552,6 +559,10 @@ module Ronin
                         Web::Spider::GitArchive.open(options[:git_archive])
                       end
 
+            history_file = if options[:history]
+                             File.open(options[:history],'w')
+                           end
+
             agent = new_agent do |agent|
               agent.every_page do |page|
                 print_page(page)
@@ -561,39 +572,12 @@ module Ronin
                 print_verbose "failed to request #{url}"
               end
 
-              if options[:print_hosts]
-                agent.every_host do |host|
-                  print_verbose "spidering new host #{host}"
-                end
-              end
-
-              if options[:print_certs]
-                agent.every_cert do |cert|
-                  print_verbose "encountered new certificate for #{cert.subject.common_name}"
-                end
-              end
-
-              if options[:print_js_strings]
-                agent.every_js_string do |string|
-                  print_content string
-                end
-              end
-
-              if options[:print_html_comments]
-                agent.every_html_comment do |comment|
-                  print_content comment
-                end
-              end
+              define_printing_callbacks(agent)
 
-              if options[:print_js_comments]
-                agent.every_js_comment do |comment|
-                  print_content comment
-                end
-              end
-
-              if options[:print_comments]
-                agent.every_comment do |comment|
-                  print_content comment
+              if history_file
+                agent.every_page do |page|
+                  history_file.puts(page.url)
+                  history_file.flush
                 end
               end
 
@@ -630,6 +614,54 @@ module Ronin
                 puts
               end
             end
+          ensure
+            if options[:history]
+              history_file.close
+            end
+          end
+
+          #
+          # Defines callbacks that print information.
+          #
+          # @param [Ronin::Web::Spider::Agent] agent
+          #   The newly created agent.
+          #
+          def define_printing_callbacks(agent)
+            if options[:print_hosts]
+              agent.every_host do |host|
+                print_verbose "spidering new host #{host}"
+              end
+            end
+
+            if options[:print_certs]
+              agent.every_cert do |cert|
+                print_verbose "encountered new certificate for #{cert.subject.common_name}"
+              end
+            end
+
+            if options[:print_js_strings]
+              agent.every_js_string do |string|
+                print_content string
+              end
+            end
+
+            if options[:print_html_comments]
+              agent.every_html_comment do |comment|
+                print_content comment
+              end
+            end
+
+            if options[:print_js_comments]
+              agent.every_js_comment do |comment|
+                print_content comment
+              end
+            end
+
+            if options[:print_comments]
+              agent.every_comment do |comment|
+                print_content comment
+              end
+            end
           end
 
           #
@@ -717,7 +749,7 @@ module Ronin
           # @param [Spidr::Page] page
           #   A spidered page.
           #
-          def print_verbose(page)
+          def print_status(page)
             if page.code < 300
               print "#{colors.bright_green(page.code)} "
             elsif page.code < 400
@@ -754,7 +786,7 @@ module Ronin
           #   A spidered page.
           #
           def print_page(page)
-            print_verbose(page) if options[:print_verbose]
+            print_status(page) if options[:print_status]
             print_url(page)
 
             if options[:print_headers]

data/lib/ronin/web/version.rb

@@ -20,6 +20,6 @@
 module Ronin
   module Web
     # ronin-web Version
-    VERSION = '1.0.0.beta3'
+    VERSION = '1.0.0.beta4'
   end
 end

data/lib/ronin/web.rb

@@ -240,7 +240,7 @@ module Ronin
     # @param [Array, Hash] parameters
     #   Additional parameters for the GET request.
     #
-    # param [Hash] headers
+    # @param [Hash] headers
     #   Additional headers for the GET request.
     #
     # @yield [page]
@@ -282,7 +282,7 @@ module Ronin
     # @param [Array, Hash] parameters
     #   Additional parameters for the GET request.
     #
-    # param [Hash] headers
+    # @param [Hash] headers
     #   Additional headers for the GET request.
     #
     # @yield [body]

data/man/ronin-web-new-webapp.1

@@ -22,7 +22,7 @@ The project directory to create\.
 .LP
 .TP
 \fB--port\fR \fIPORT\fP
-The port the webpp will listen on by default\. Defaults to \fB3000\fR\.
+The port the webapp will listen on by default\. Defaults to \fB3000\fR\.
 .LP
 .TP
 \fB--ruby-version\fR \fIVERSION\fP

data/man/ronin-web-new-webapp.1.md

@@ -16,7 +16,7 @@ Generates a new `ronin-web-server` based webapp.
 ## OPTIONS
 
 `--port` *PORT*
-  The port the webpp will listen on by default. Defaults to `3000`.
+  The port the webapp will listen on by default. Defaults to `3000`.
 
 `--ruby-version` *VERSION*
   The desired ruby version for the project Defaults to the current ruby version.

data/man/ronin-web-spider.1

@@ -179,6 +179,10 @@ Print response headers for each URL\.
 Prints a specific header\.
 .LP
 .TP
+\fB--history\fR \fIFILE\fP
+Sets the history file to write every visited URL to\.
+.LP
+.TP
 \fB--archive\fR \fIDIR\fP
 Archive every visited page to the \fIDIR\fP\.
 .LP

data/man/ronin-web-spider.1.md

@@ -133,6 +133,9 @@ Spiders a website.
 `--print-header` *NAME*
   Prints a specific header.
 
+`--history` *FILE*
+  Sets the history file to write every visited URL to.
+
 `--archive` *DIR*
   Archive every visited page to the *DIR*.
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ronin-web
 version: !ruby/object:Gem::Version
-  version: 1.0.0.beta3
+  version: 1.0.0.beta4
 platform: ruby
 authors:
 - Postmodern
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-01-14 00:00:00.000000000 Z
+date: 2023-01-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri