ronin-web 1.0.0.beta3 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ruby.yml +1 -0
- data/README.md +1 -0
- data/gemspec.yml +5 -5
- data/lib/ronin/web/cli/commands/spider.rb +67 -35
- data/lib/ronin/web/version.rb +1 -1
- data/lib/ronin/web.rb +2 -2
- data/man/ronin-web-new-webapp.1 +1 -1
- data/man/ronin-web-new-webapp.1.md +1 -1
- data/man/ronin-web-spider.1 +4 -0
- data/man/ronin-web-spider.1.md +3 -0
- metadata +12 -12
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b096e4cbd0c20e6ff15103dd87681a23a1253f1da9af3bf6b74aac474082fcf2
|
4
|
+
data.tar.gz: bb0b34acff8be4bf2b7a89522c7acc593df7adea843f258b616596dd07632423
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 798cfd3b63bcc624b9f79819794923bad26c4127a0b0fbc8f28d55fe02afaf2f90cf070f265afc6360e29a35d1afcc8a62f0721ad0256ff1accb86a3cfb8f6ee
|
7
|
+
data.tar.gz: f87af34ce8994118a453ac1ec672e5af5f140ee916271291fa23a194d7d87f80f906a87fefe8bf82e2453aad3a2fed60b267729e8de6330419dca916bcc13f8f
|
data/.github/workflows/ruby.yml
CHANGED
data/README.md
CHANGED
@@ -2,6 +2,7 @@
|
|
2
2
|
|
3
3
|
[![CI](https://github.com/ronin-rb/ronin-web/actions/workflows/ruby.yml/badge.svg)](https://github.com/ronin-rb/ronin-web/actions/workflows/ruby.yml)
|
4
4
|
[![Code Climate](https://codeclimate.com/github/ronin-rb/ronin-web.svg)](https://codeclimate.com/github/ronin-rb/ronin-web)
|
5
|
+
[![Gem Version](https://badge.fury.io/rb/ronin-web.svg)](https://badge.fury.io/rb/ronin-web)
|
5
6
|
|
6
7
|
* [Source](https://github.com/ronin-rb/ronin-web)
|
7
8
|
* [Issues](https://github.com/ronin-rb/ronin-web/issues)
|
data/gemspec.yml
CHANGED
@@ -40,11 +40,11 @@ dependencies:
|
|
40
40
|
mechanize: ~> 2.0
|
41
41
|
open_namespace: ~> 0.4
|
42
42
|
# Ronin dependencies:
|
43
|
-
ronin-support: ~> 1.0
|
44
|
-
ronin-web-server: ~> 0.1
|
45
|
-
ronin-web-spider: ~> 0.1
|
46
|
-
ronin-web-user_agents: ~> 0.1
|
47
|
-
ronin-core: ~> 0.1
|
43
|
+
ronin-support: ~> 1.0
|
44
|
+
ronin-web-server: ~> 0.1
|
45
|
+
ronin-web-spider: ~> 0.1
|
46
|
+
ronin-web-user_agents: ~> 0.1
|
47
|
+
ronin-core: ~> 0.1
|
48
48
|
|
49
49
|
development_dependencies:
|
50
50
|
bundler: ~> 2.0
|
@@ -83,6 +83,7 @@ module Ronin
|
|
83
83
|
# --print-status Print the status codes for each URL
|
84
84
|
# --print-headers Print response headers for each URL
|
85
85
|
# --print-header NAME Prints a specific header
|
86
|
+
# --history FILE The history file
|
86
87
|
# --archive DIR Archive every visited page to the DIR
|
87
88
|
# --git-archive DIR Archive every visited page to the git repository
|
88
89
|
# -X, --xpath XPATH Evaluates the XPath on each HTML page
|
@@ -390,7 +391,7 @@ module Ronin
|
|
390
391
|
},
|
391
392
|
desc: 'Spiders the website, starting at the URL'
|
392
393
|
|
393
|
-
option :
|
394
|
+
option :print_stauts, desc: 'Print the status codes for each URL'
|
394
395
|
|
395
396
|
option :print_headers, desc: 'Print response headers for each URL'
|
396
397
|
|
@@ -400,6 +401,12 @@ module Ronin
|
|
400
401
|
},
|
401
402
|
desc: 'Prints a specific header'
|
402
403
|
|
404
|
+
option :history, value: {
|
405
|
+
type: String,
|
406
|
+
usage: 'FILE'
|
407
|
+
},
|
408
|
+
desc: 'The history file'
|
409
|
+
|
403
410
|
option :archive, value: {
|
404
411
|
type: String,
|
405
412
|
usage: 'DIR'
|
@@ -552,6 +559,10 @@ module Ronin
|
|
552
559
|
Web::Spider::GitArchive.open(options[:git_archive])
|
553
560
|
end
|
554
561
|
|
562
|
+
history_file = if options[:history]
|
563
|
+
File.open(options[:history],'w')
|
564
|
+
end
|
565
|
+
|
555
566
|
agent = new_agent do |agent|
|
556
567
|
agent.every_page do |page|
|
557
568
|
print_page(page)
|
@@ -561,39 +572,12 @@ module Ronin
|
|
561
572
|
print_verbose "failed to request #{url}"
|
562
573
|
end
|
563
574
|
|
564
|
-
|
565
|
-
agent.every_host do |host|
|
566
|
-
print_verbose "spidering new host #{host}"
|
567
|
-
end
|
568
|
-
end
|
569
|
-
|
570
|
-
if options[:print_certs]
|
571
|
-
agent.every_cert do |cert|
|
572
|
-
print_verbose "encountered new certificate for #{cert.subject.common_name}"
|
573
|
-
end
|
574
|
-
end
|
575
|
-
|
576
|
-
if options[:print_js_strings]
|
577
|
-
agent.every_js_string do |string|
|
578
|
-
print_content string
|
579
|
-
end
|
580
|
-
end
|
581
|
-
|
582
|
-
if options[:print_html_comments]
|
583
|
-
agent.every_html_comment do |comment|
|
584
|
-
print_content comment
|
585
|
-
end
|
586
|
-
end
|
575
|
+
define_printing_callbacks(agent)
|
587
576
|
|
588
|
-
if
|
589
|
-
agent.
|
590
|
-
|
591
|
-
|
592
|
-
end
|
593
|
-
|
594
|
-
if options[:print_comments]
|
595
|
-
agent.every_comment do |comment|
|
596
|
-
print_content comment
|
577
|
+
if history_file
|
578
|
+
agent.every_page do |page|
|
579
|
+
history_file.puts(page.url)
|
580
|
+
history_file.flush
|
597
581
|
end
|
598
582
|
end
|
599
583
|
|
@@ -630,6 +614,54 @@ module Ronin
|
|
630
614
|
puts
|
631
615
|
end
|
632
616
|
end
|
617
|
+
ensure
|
618
|
+
if options[:history]
|
619
|
+
history_file.close
|
620
|
+
end
|
621
|
+
end
|
622
|
+
|
623
|
+
#
|
624
|
+
# Defines callbacks that print information.
|
625
|
+
#
|
626
|
+
# @param [Ronin::Web::Spider::Agent] agent
|
627
|
+
# The newly created agent.
|
628
|
+
#
|
629
|
+
def define_printing_callbacks(agent)
|
630
|
+
if options[:print_hosts]
|
631
|
+
agent.every_host do |host|
|
632
|
+
print_verbose "spidering new host #{host}"
|
633
|
+
end
|
634
|
+
end
|
635
|
+
|
636
|
+
if options[:print_certs]
|
637
|
+
agent.every_cert do |cert|
|
638
|
+
print_verbose "encountered new certificate for #{cert.subject.common_name}"
|
639
|
+
end
|
640
|
+
end
|
641
|
+
|
642
|
+
if options[:print_js_strings]
|
643
|
+
agent.every_js_string do |string|
|
644
|
+
print_content string
|
645
|
+
end
|
646
|
+
end
|
647
|
+
|
648
|
+
if options[:print_html_comments]
|
649
|
+
agent.every_html_comment do |comment|
|
650
|
+
print_content comment
|
651
|
+
end
|
652
|
+
end
|
653
|
+
|
654
|
+
if options[:print_js_comments]
|
655
|
+
agent.every_js_comment do |comment|
|
656
|
+
print_content comment
|
657
|
+
end
|
658
|
+
end
|
659
|
+
|
660
|
+
if options[:print_comments]
|
661
|
+
agent.every_comment do |comment|
|
662
|
+
print_content comment
|
663
|
+
end
|
664
|
+
end
|
633
665
|
end
|
634
666
|
|
635
667
|
#
|
@@ -717,7 +749,7 @@ module Ronin
|
|
717
749
|
# @param [Spidr::Page] page
|
718
750
|
# A spidered page.
|
719
751
|
#
|
720
|
-
def
|
752
|
+
def print_status(page)
|
721
753
|
if page.code < 300
|
722
754
|
print "#{colors.bright_green(page.code)} "
|
723
755
|
elsif page.code < 400
|
@@ -754,7 +786,7 @@ module Ronin
|
|
754
786
|
# A spidered page.
|
755
787
|
#
|
756
788
|
def print_page(page)
|
757
|
-
|
789
|
+
print_status(page) if options[:print_status]
|
758
790
|
print_url(page)
|
759
791
|
|
760
792
|
if options[:print_headers]
|
data/lib/ronin/web/version.rb
CHANGED
data/lib/ronin/web.rb
CHANGED
@@ -240,7 +240,7 @@ module Ronin
|
|
240
240
|
# @param [Array, Hash] parameters
|
241
241
|
# Additional parameters for the GET request.
|
242
242
|
#
|
243
|
-
# param [Hash] headers
|
243
|
+
# @param [Hash] headers
|
244
244
|
# Additional headers for the GET request.
|
245
245
|
#
|
246
246
|
# @yield [page]
|
@@ -282,7 +282,7 @@ module Ronin
|
|
282
282
|
# @param [Array, Hash] parameters
|
283
283
|
# Additional parameters for the GET request.
|
284
284
|
#
|
285
|
-
# param [Hash] headers
|
285
|
+
# @param [Hash] headers
|
286
286
|
# Additional headers for the GET request.
|
287
287
|
#
|
288
288
|
# @yield [body]
|
data/man/ronin-web-new-webapp.1
CHANGED
@@ -22,7 +22,7 @@ The project directory to create\.
|
|
22
22
|
.LP
|
23
23
|
.TP
|
24
24
|
\fB--port\fR \fIPORT\fP
|
25
|
-
The port the
|
25
|
+
The port the webapp will listen on by default\. Defaults to \fB3000\fR\.
|
26
26
|
.LP
|
27
27
|
.TP
|
28
28
|
\fB--ruby-version\fR \fIVERSION\fP
|
@@ -16,7 +16,7 @@ Generates a new `ronin-web-server` based webapp.
|
|
16
16
|
## OPTIONS
|
17
17
|
|
18
18
|
`--port` *PORT*
|
19
|
-
The port the
|
19
|
+
The port the webapp will listen on by default. Defaults to `3000`.
|
20
20
|
|
21
21
|
`--ruby-version` *VERSION*
|
22
22
|
The desired ruby version for the project Defaults to the current ruby version.
|
data/man/ronin-web-spider.1
CHANGED
@@ -179,6 +179,10 @@ Print response headers for each URL\.
|
|
179
179
|
Prints a specific header\.
|
180
180
|
.LP
|
181
181
|
.TP
|
182
|
+
\fB--history\fR \fIFILE\fP
|
183
|
+
Sets the history file to write every visited URL to\.
|
184
|
+
.LP
|
185
|
+
.TP
|
182
186
|
\fB--archive\fR \fIDIR\fP
|
183
187
|
Archive every visited page to the \fIDIR\fP\.
|
184
188
|
.LP
|
data/man/ronin-web-spider.1.md
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ronin-web
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.0
|
4
|
+
version: 1.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Postmodern
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-01
|
11
|
+
date: 2023-02-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: nokogiri
|
@@ -86,70 +86,70 @@ dependencies:
|
|
86
86
|
requirements:
|
87
87
|
- - "~>"
|
88
88
|
- !ruby/object:Gem::Version
|
89
|
-
version: 1.0
|
89
|
+
version: '1.0'
|
90
90
|
type: :runtime
|
91
91
|
prerelease: false
|
92
92
|
version_requirements: !ruby/object:Gem::Requirement
|
93
93
|
requirements:
|
94
94
|
- - "~>"
|
95
95
|
- !ruby/object:Gem::Version
|
96
|
-
version: 1.0
|
96
|
+
version: '1.0'
|
97
97
|
- !ruby/object:Gem::Dependency
|
98
98
|
name: ronin-web-server
|
99
99
|
requirement: !ruby/object:Gem::Requirement
|
100
100
|
requirements:
|
101
101
|
- - "~>"
|
102
102
|
- !ruby/object:Gem::Version
|
103
|
-
version: 0.1
|
103
|
+
version: '0.1'
|
104
104
|
type: :runtime
|
105
105
|
prerelease: false
|
106
106
|
version_requirements: !ruby/object:Gem::Requirement
|
107
107
|
requirements:
|
108
108
|
- - "~>"
|
109
109
|
- !ruby/object:Gem::Version
|
110
|
-
version: 0.1
|
110
|
+
version: '0.1'
|
111
111
|
- !ruby/object:Gem::Dependency
|
112
112
|
name: ronin-web-spider
|
113
113
|
requirement: !ruby/object:Gem::Requirement
|
114
114
|
requirements:
|
115
115
|
- - "~>"
|
116
116
|
- !ruby/object:Gem::Version
|
117
|
-
version: 0.1
|
117
|
+
version: '0.1'
|
118
118
|
type: :runtime
|
119
119
|
prerelease: false
|
120
120
|
version_requirements: !ruby/object:Gem::Requirement
|
121
121
|
requirements:
|
122
122
|
- - "~>"
|
123
123
|
- !ruby/object:Gem::Version
|
124
|
-
version: 0.1
|
124
|
+
version: '0.1'
|
125
125
|
- !ruby/object:Gem::Dependency
|
126
126
|
name: ronin-web-user_agents
|
127
127
|
requirement: !ruby/object:Gem::Requirement
|
128
128
|
requirements:
|
129
129
|
- - "~>"
|
130
130
|
- !ruby/object:Gem::Version
|
131
|
-
version: 0.1
|
131
|
+
version: '0.1'
|
132
132
|
type: :runtime
|
133
133
|
prerelease: false
|
134
134
|
version_requirements: !ruby/object:Gem::Requirement
|
135
135
|
requirements:
|
136
136
|
- - "~>"
|
137
137
|
- !ruby/object:Gem::Version
|
138
|
-
version: 0.1
|
138
|
+
version: '0.1'
|
139
139
|
- !ruby/object:Gem::Dependency
|
140
140
|
name: ronin-core
|
141
141
|
requirement: !ruby/object:Gem::Requirement
|
142
142
|
requirements:
|
143
143
|
- - "~>"
|
144
144
|
- !ruby/object:Gem::Version
|
145
|
-
version: 0.1
|
145
|
+
version: '0.1'
|
146
146
|
type: :runtime
|
147
147
|
prerelease: false
|
148
148
|
version_requirements: !ruby/object:Gem::Requirement
|
149
149
|
requirements:
|
150
150
|
- - "~>"
|
151
151
|
- !ruby/object:Gem::Version
|
152
|
-
version: 0.1
|
152
|
+
version: '0.1'
|
153
153
|
- !ruby/object:Gem::Dependency
|
154
154
|
name: bundler
|
155
155
|
requirement: !ruby/object:Gem::Requirement
|