ronin-web 1.0.0.beta3 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4e36e87f767a665ff3dafb90022b8cb0b0d8d7d9ebc23255c873dcebde04d7b1
-  data.tar.gz: af5f942251ddefcfd8940d2de5cc688edbd6e55ba96715d8798b0e85efff33f6
+  metadata.gz: b096e4cbd0c20e6ff15103dd87681a23a1253f1da9af3bf6b74aac474082fcf2
+  data.tar.gz: bb0b34acff8be4bf2b7a89522c7acc593df7adea843f258b616596dd07632423
 SHA512:
-  metadata.gz: 9e0bc704b1177e0f958c00a0f1f33d606258c78186dc2ac38de85d7188d29e9911aab5d337cb39a7e90798865841b947abd8c6e928325df2a394e7a61b6576a2
-  data.tar.gz: c7dd821942055424f990ab2f5511252308d9b12bf3bdb2ccb07281d523d40066425989d1645a27f464321147973b1ff12410c6bcc79cece614d09efbaf730e57
+  metadata.gz: 798cfd3b63bcc624b9f79819794923bad26c4127a0b0fbc8f28d55fe02afaf2f90cf070f265afc6360e29a35d1afcc8a62f0721ad0256ff1accb86a3cfb8f6ee
+  data.tar.gz: f87af34ce8994118a453ac1ec672e5af5f140ee916271291fa23a194d7d87f80f906a87fefe8bf82e2453aad3a2fed60b267729e8de6330419dca916bcc13f8f

data/.github/workflows/ruby.yml

@@ -21,6 +21,7 @@ jobs:
         uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
       - name: Install dependencies
         run: bundle install --jobs 4 --retry 3
       - name: Run tests

data/README.md

@@ -2,6 +2,7 @@
 
 [![CI](https://github.com/ronin-rb/ronin-web/actions/workflows/ruby.yml/badge.svg)](https://github.com/ronin-rb/ronin-web/actions/workflows/ruby.yml)
 [![Code Climate](https://codeclimate.com/github/ronin-rb/ronin-web.svg)](https://codeclimate.com/github/ronin-rb/ronin-web)
+[![Gem Version](https://badge.fury.io/rb/ronin-web.svg)](https://badge.fury.io/rb/ronin-web)
 
 * [Source](https://github.com/ronin-rb/ronin-web)
 * [Issues](https://github.com/ronin-rb/ronin-web/issues)

data/gemspec.yml

@@ -40,11 +40,11 @@ dependencies:
   mechanize: ~> 2.0
   open_namespace: ~> 0.4
   # Ronin dependencies:
-  ronin-support: ~> 1.0.0.beta2
-  ronin-web-server: ~> 0.1.0.beta3
-  ronin-web-spider: ~> 0.1.0.beta2
-  ronin-web-user_agents: ~> 0.1.0.beta1
-  ronin-core: ~> 0.1.0.beta1
+  ronin-support: ~> 1.0
+  ronin-web-server: ~> 0.1
+  ronin-web-spider: ~> 0.1
+  ronin-web-user_agents: ~> 0.1
+  ronin-core: ~> 0.1
 
 development_dependencies:
   bundler: ~> 2.0

data/lib/ronin/web/cli/commands/spider.rb

@@ -83,6 +83,7 @@ module Ronin
         #         --print-status               Print the status codes for each URL
         #         --print-headers              Print response headers for each URL
         #         --print-header NAME          Prints a specific header
+        #         --history FILE               The history file
         #         --archive DIR                Archive every visited page to the DIR
         #         --git-archive DIR            Archive every visited page to the git repository
         #     -X, --xpath XPATH                Evaluates the XPath on each HTML page
@@ -390,7 +391,7 @@ module Ronin
                         },
                         desc: 'Spiders the website, starting at the URL'
 
-          option :print_verbose, desc: 'Print the status codes for each URL'
+          option :print_stauts, desc: 'Print the status codes for each URL'
 
           option :print_headers, desc: 'Print response headers for each URL'
 
@@ -400,6 +401,12 @@ module Ronin
                                 },
                                 desc: 'Prints a specific header'
 
+          option :history, value: {
+                             type: String,
+                             usage: 'FILE'
+                           },
+                           desc: 'The history file'
+
           option :archive, value: {
                              type: String,
                              usage: 'DIR'
@@ -552,6 +559,10 @@ module Ronin
                         Web::Spider::GitArchive.open(options[:git_archive])
                       end
 
+            history_file = if options[:history]
+                             File.open(options[:history],'w')
+                           end
+
             agent = new_agent do |agent|
               agent.every_page do |page|
                 print_page(page)
@@ -561,39 +572,12 @@ module Ronin
                 print_verbose "failed to request #{url}"
               end
 
-              if options[:print_hosts]
-                agent.every_host do |host|
-                  print_verbose "spidering new host #{host}"
-                end
-              end
-
-              if options[:print_certs]
-                agent.every_cert do |cert|
-                  print_verbose "encountered new certificate for #{cert.subject.common_name}"
-                end
-              end
-
-              if options[:print_js_strings]
-                agent.every_js_string do |string|
-                  print_content string
-                end
-              end
-
-              if options[:print_html_comments]
-                agent.every_html_comment do |comment|
-                  print_content comment
-                end
-              end
+              define_printing_callbacks(agent)
 
-              if options[:print_js_comments]
-                agent.every_js_comment do |comment|
-                  print_content comment
-                end
-              end
-
-              if options[:print_comments]
-                agent.every_comment do |comment|
-                  print_content comment
+              if history_file
+                agent.every_page do |page|
+                  history_file.puts(page.url)
+                  history_file.flush
                 end
               end
 
@@ -630,6 +614,54 @@ module Ronin
                 puts
               end
             end
+          ensure
+            if options[:history]
+              history_file.close
+            end
+          end
+
+          #
+          # Defines callbacks that print information.
+          #
+          # @param [Ronin::Web::Spider::Agent] agent
+          #   The newly created agent.
+          #
+          def define_printing_callbacks(agent)
+            if options[:print_hosts]
+              agent.every_host do |host|
+                print_verbose "spidering new host #{host}"
+              end
+            end
+
+            if options[:print_certs]
+              agent.every_cert do |cert|
+                print_verbose "encountered new certificate for #{cert.subject.common_name}"
+              end
+            end
+
+            if options[:print_js_strings]
+              agent.every_js_string do |string|
+                print_content string
+              end
+            end
+
+            if options[:print_html_comments]
+              agent.every_html_comment do |comment|
+                print_content comment
+              end
+            end
+
+            if options[:print_js_comments]
+              agent.every_js_comment do |comment|
+                print_content comment
+              end
+            end
+
+            if options[:print_comments]
+              agent.every_comment do |comment|
+                print_content comment
+              end
+            end
           end
 
           #
@@ -717,7 +749,7 @@ module Ronin
           # @param [Spidr::Page] page
           #   A spidered page.
           #
-          def print_verbose(page)
+          def print_status(page)
             if page.code < 300
               print "#{colors.bright_green(page.code)} "
             elsif page.code < 400
@@ -754,7 +786,7 @@ module Ronin
           #   A spidered page.
           #
           def print_page(page)
-            print_verbose(page) if options[:print_verbose]
+            print_status(page) if options[:print_status]
             print_url(page)
 
             if options[:print_headers]

data/lib/ronin/web/version.rb

@@ -20,6 +20,6 @@
 module Ronin
   module Web
     # ronin-web Version
-    VERSION = '1.0.0.beta3'
+    VERSION = '1.0.0'
   end
 end

data/lib/ronin/web.rb

@@ -240,7 +240,7 @@ module Ronin
     # @param [Array, Hash] parameters
     #   Additional parameters for the GET request.
     #
-    # param [Hash] headers
+    # @param [Hash] headers
     #   Additional headers for the GET request.
     #
     # @yield [page]
@@ -282,7 +282,7 @@ module Ronin
     # @param [Array, Hash] parameters
     #   Additional parameters for the GET request.
     #
-    # param [Hash] headers
+    # @param [Hash] headers
     #   Additional headers for the GET request.
     #
     # @yield [body]

data/man/ronin-web-new-webapp.1

@@ -22,7 +22,7 @@ The project directory to create\.
 .LP
 .TP
 \fB--port\fR \fIPORT\fP
-The port the webpp will listen on by default\. Defaults to \fB3000\fR\.
+The port the webapp will listen on by default\. Defaults to \fB3000\fR\.
 .LP
 .TP
 \fB--ruby-version\fR \fIVERSION\fP

data/man/ronin-web-new-webapp.1.md

@@ -16,7 +16,7 @@ Generates a new `ronin-web-server` based webapp.
 ## OPTIONS
 
 `--port` *PORT*
-  The port the webpp will listen on by default. Defaults to `3000`.
+  The port the webapp will listen on by default. Defaults to `3000`.
 
 `--ruby-version` *VERSION*
   The desired ruby version for the project Defaults to the current ruby version.

data/man/ronin-web-spider.1

@@ -179,6 +179,10 @@ Print response headers for each URL\.
 Prints a specific header\.
 .LP
 .TP
+\fB--history\fR \fIFILE\fP
+Sets the history file to write every visited URL to\.
+.LP
+.TP
 \fB--archive\fR \fIDIR\fP
 Archive every visited page to the \fIDIR\fP\.
 .LP

data/man/ronin-web-spider.1.md

@@ -133,6 +133,9 @@ Spiders a website.
 `--print-header` *NAME*
   Prints a specific header.
 
+`--history` *FILE*
+  Sets the history file to write every visited URL to.
+
 `--archive` *DIR*
   Archive every visited page to the *DIR*.
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ronin-web
 version: !ruby/object:Gem::Version
-  version: 1.0.0.beta3
+  version: 1.0.0
 platform: ruby
 authors:
 - Postmodern
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-01-14 00:00:00.000000000 Z
+date: 2023-02-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -86,70 +86,70 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.0.beta2
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.0.beta2
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: ronin-web-server
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.0.beta3
+        version: '0.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.0.beta3
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: ronin-web-spider
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.0.beta2
+        version: '0.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.0.beta2
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: ronin-web-user_agents
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.0.beta1
+        version: '0.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.0.beta1
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: ronin-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.0.beta1
+        version: '0.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.0.beta1
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement