ronin-web 0.1.3 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data.tar.gz.sig +0 -0
- data/History.txt +25 -0
- data/Manifest.txt +36 -4
- data/README.txt +67 -64
- data/Rakefile +12 -3
- data/bin/ronin-web +1 -1
- data/lib/ronin/network/helpers/web.rb +221 -0
- data/lib/ronin/web.rb +1 -2
- data/lib/ronin/web/extensions.rb +0 -2
- data/lib/ronin/web/extensions/nokogiri.rb +0 -23
- data/lib/ronin/web/proxy.rb +3 -103
- data/lib/ronin/web/proxy/app.rb +31 -0
- data/lib/ronin/web/proxy/base.rb +41 -0
- data/lib/ronin/web/proxy/web.rb +42 -0
- data/lib/ronin/web/server.rb +3 -530
- data/lib/ronin/web/server/app.rb +31 -0
- data/lib/ronin/web/server/base.rb +334 -0
- data/lib/ronin/web/server/files.rb +92 -0
- data/lib/ronin/web/server/helpers.rb +25 -0
- data/lib/ronin/web/server/helpers/files.rb +126 -0
- data/lib/ronin/web/server/helpers/hosts.rb +72 -0
- data/lib/ronin/web/server/helpers/proxy.rb +153 -0
- data/lib/ronin/web/server/helpers/rendering.rb +36 -0
- data/lib/ronin/web/server/hosts.rb +86 -0
- data/lib/ronin/web/server/proxy.rb +116 -0
- data/lib/ronin/web/server/web.rb +62 -0
- data/lib/ronin/web/spider.rb +53 -26
- data/lib/ronin/web/version.rb +1 -3
- data/lib/ronin/web/web.rb +253 -95
- data/spec/spec_helper.rb +1 -1
- data/spec/web/proxy/base_spec.rb +9 -0
- data/spec/web/server/base_spec.rb +86 -0
- data/spec/web/server/classes/files/dir/file.txt +1 -0
- data/spec/web/server/classes/files/dir/index.html +1 -0
- data/spec/web/server/classes/files/dir2/file2.txt +1 -0
- data/spec/web/server/classes/files/dir3/page.xml +4 -0
- data/spec/web/server/classes/files/file.txt +1 -0
- data/spec/web/server/classes/files_app.rb +27 -0
- data/spec/web/server/classes/hosts_app.rb +40 -0
- data/spec/web/server/classes/proxy_app.rb +45 -0
- data/spec/web/server/classes/public1/static1.txt +1 -0
- data/spec/web/server/classes/public2/static2.txt +1 -0
- data/spec/web/server/classes/sub_app.rb +13 -0
- data/spec/web/server/classes/test_app.rb +20 -0
- data/spec/web/server/files_spec.rb +74 -0
- data/spec/web/server/helpers/server.rb +42 -0
- data/spec/web/server/hosts_spec.rb +55 -0
- data/spec/web/server/proxy_spec.rb +49 -0
- data/tasks/spec.rb +1 -0
- data/tasks/yard.rb +13 -0
- metadata +76 -17
- metadata.gz.sig +0 -0
- data/TODO.txt +0 -7
- data/lib/ronin/sessions/web.rb +0 -80
- data/lib/ronin/web/fingerprint.rb +0 -76
- data/spec/web/server_spec.rb +0 -142
@@ -0,0 +1,116 @@
|
|
1
|
+
#
|
2
|
+
# Ronin Web - A Ruby library for Ronin that provides support for web
|
3
|
+
# scraping and spidering functionality.
|
4
|
+
#
|
5
|
+
# Copyright (c) 2006-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
|
6
|
+
#
|
7
|
+
# This program is free software; you can redistribute it and/or modify
|
8
|
+
# it under the terms of the GNU General Public License as published by
|
9
|
+
# the Free Software Foundation; either version 2 of the License, or
|
10
|
+
# (at your option) any later version.
|
11
|
+
#
|
12
|
+
# This program is distributed in the hope that it will be useful,
|
13
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
14
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
15
|
+
# GNU General Public License for more details.
|
16
|
+
#
|
17
|
+
# You should have received a copy of the GNU General Public License
|
18
|
+
# along with this program; if not, write to the Free Software
|
19
|
+
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
|
20
|
+
#
|
21
|
+
|
22
|
+
require 'ronin/web/server/helpers/proxy'
|
23
|
+
|
24
|
+
module Ronin
|
25
|
+
module Web
|
26
|
+
module Server
|
27
|
+
module Proxy
|
28
|
+
def self.included(base)
|
29
|
+
base.module_eval do
|
30
|
+
#
|
31
|
+
# Proxies requests to a given path.
|
32
|
+
#
|
33
|
+
# @param [String] path
|
34
|
+
# The path to proxy requests for.
|
35
|
+
#
|
36
|
+
# @param [Hash] options
|
37
|
+
# Additional options.
|
38
|
+
#
|
39
|
+
# @yield [(response), body]
|
40
|
+
# If a block is given, it will be passed the optional
|
41
|
+
# response of the proxied request and the body received
|
42
|
+
# from the proxied request.
|
43
|
+
#
|
44
|
+
# @yieldparam [Net::HTTP::Response] response
|
45
|
+
# The response.
|
46
|
+
#
|
47
|
+
# @yieldparam [String] body
|
48
|
+
# The body from the response.
|
49
|
+
#
|
50
|
+
# @example
|
51
|
+
# proxy '/login.php' do |body|
|
52
|
+
# body.gsub(/https/,'http')
|
53
|
+
# end
|
54
|
+
#
|
55
|
+
# @example
|
56
|
+
# proxy '/login*' do |response,body|
|
57
|
+
# end
|
58
|
+
#
|
59
|
+
# @since 0.2.0
|
60
|
+
#
|
61
|
+
def self.proxy(path,options={},&block)
|
62
|
+
any(path) do
|
63
|
+
proxy(options,&block)
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
#
|
68
|
+
# Proxies requests to a given path.
|
69
|
+
#
|
70
|
+
# @param [String] path
|
71
|
+
# The path to proxy requests for.
|
72
|
+
#
|
73
|
+
# @param [Hash] options
|
74
|
+
# Additional options.
|
75
|
+
#
|
76
|
+
# @yield [(response), page]
|
77
|
+
# If a block is given, it will be passed the optional
|
78
|
+
# response of the proxied request and the page from the
|
79
|
+
# proxied request.
|
80
|
+
#
|
81
|
+
# @yieldparam [Net::HTTP::Response] response
|
82
|
+
# The response.
|
83
|
+
#
|
84
|
+
# @yieldparam [Nokogiri::HTML, Nokogiri::XML] page
|
85
|
+
# The page from the response.
|
86
|
+
#
|
87
|
+
# @example
|
88
|
+
# proxy '/login.php' do |page|
|
89
|
+
# body.search('@action').each do |action|
|
90
|
+
# action.inner_text = action.inner_text.gsub(
|
91
|
+
# /https/, 'http'
|
92
|
+
# )
|
93
|
+
# end
|
94
|
+
# end
|
95
|
+
#
|
96
|
+
# @example
|
97
|
+
# proxy '/login*' do |response,body|
|
98
|
+
# end
|
99
|
+
#
|
100
|
+
# @since 0.2.0
|
101
|
+
#
|
102
|
+
def self.proxy_page(path,options={},&block)
|
103
|
+
any(path) do
|
104
|
+
proxy_page(options,&block)
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
protected
|
109
|
+
|
110
|
+
helpers Ronin::Web::Server::Helpers::Proxy
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
115
|
+
end
|
116
|
+
end
|
@@ -0,0 +1,62 @@
|
|
1
|
+
#
|
2
|
+
# Ronin Web - A Ruby library for Ronin that provides support for web
|
3
|
+
# scraping and spidering functionality.
|
4
|
+
#
|
5
|
+
# Copyright (c) 2006-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
|
6
|
+
#
|
7
|
+
# This program is free software; you can redistribute it and/or modify
|
8
|
+
# it under the terms of the GNU General Public License as published by
|
9
|
+
# the Free Software Foundation; either version 2 of the License, or
|
10
|
+
# (at your option) any later version.
|
11
|
+
#
|
12
|
+
# This program is distributed in the hope that it will be useful,
|
13
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
14
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
15
|
+
# GNU General Public License for more details.
|
16
|
+
#
|
17
|
+
# You should have received a copy of the GNU General Public License
|
18
|
+
# along with this program; if not, write to the Free Software
|
19
|
+
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
|
20
|
+
#
|
21
|
+
|
22
|
+
require 'ronin/web/server/app'
|
23
|
+
|
24
|
+
module Ronin
|
25
|
+
module Web
|
26
|
+
#
|
27
|
+
# Returns the Ronin Web Server.
|
28
|
+
#
|
29
|
+
# @param [Hash] options
|
30
|
+
# Additional options.
|
31
|
+
#
|
32
|
+
# @yield [server]
|
33
|
+
# If a block is given, it will be passed the current web server.
|
34
|
+
#
|
35
|
+
# @yieldparam [Server::App]
|
36
|
+
# The current web server class.
|
37
|
+
#
|
38
|
+
# @return [Server::App]
|
39
|
+
# The current web server class.
|
40
|
+
#
|
41
|
+
# @example
|
42
|
+
# Web.server do
|
43
|
+
# get '/hello' do
|
44
|
+
# 'world'
|
45
|
+
# end
|
46
|
+
# end
|
47
|
+
#
|
48
|
+
# @see Server::Base.run!
|
49
|
+
# @since 0.2.0
|
50
|
+
#
|
51
|
+
def Web.server(options={},&block)
|
52
|
+
unless class_variable_defined?('@@ronin_web_server')
|
53
|
+
@@ronin_web_server = Server::App
|
54
|
+
@@ronin_web_server.run!(options.merge(:background => true))
|
55
|
+
end
|
56
|
+
|
57
|
+
@@ronin_web_server.class_eval(&block)
|
58
|
+
|
59
|
+
return @@ronin_web_server
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
data/lib/ronin/web/spider.rb
CHANGED
@@ -1,5 +1,4 @@
|
|
1
1
|
#
|
2
|
-
#--
|
3
2
|
# Ronin Web - A Ruby library for Ronin that provides support for web
|
4
3
|
# scraping and spidering functionality.
|
5
4
|
#
|
@@ -18,11 +17,10 @@
|
|
18
17
|
# You should have received a copy of the GNU General Public License
|
19
18
|
# along with this program; if not, write to the Free Software
|
20
19
|
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
|
21
|
-
#++
|
22
20
|
#
|
23
21
|
|
24
22
|
require 'ronin/web/web'
|
25
|
-
require 'ronin/ui/
|
23
|
+
require 'ronin/ui/output/helpers'
|
26
24
|
|
27
25
|
require 'spidr/agent'
|
28
26
|
|
@@ -30,31 +28,60 @@ module Ronin
|
|
30
28
|
module Web
|
31
29
|
class Spider < Spidr::Agent
|
32
30
|
|
33
|
-
include UI::
|
31
|
+
include UI::Output::Helpers
|
34
32
|
|
35
33
|
#
|
36
|
-
# Creates a new Spider object
|
37
|
-
#
|
38
|
-
#
|
39
|
-
#
|
40
|
-
#
|
41
|
-
#
|
42
|
-
#
|
43
|
-
#
|
44
|
-
#
|
45
|
-
#
|
46
|
-
#
|
47
|
-
#
|
48
|
-
#
|
49
|
-
#
|
50
|
-
#
|
51
|
-
#
|
52
|
-
#
|
53
|
-
#
|
54
|
-
#
|
55
|
-
#
|
56
|
-
#
|
57
|
-
#
|
34
|
+
# Creates a new Spider object.
|
35
|
+
#
|
36
|
+
# @param [Hash] options
|
37
|
+
# Additional options.
|
38
|
+
#
|
39
|
+
# @option options [Hash] :proxy (Web.proxy)
|
40
|
+
# The proxy to use while spidering.
|
41
|
+
#
|
42
|
+
# @option options [String] :user_agent
|
43
|
+
# The User-Agent string to send.
|
44
|
+
#
|
45
|
+
# @option options [String] :referer
|
46
|
+
# The referer URL to send.
|
47
|
+
#
|
48
|
+
# @option options [Integer] :delay (0)
|
49
|
+
# Duration in seconds to pause between spidering each link.
|
50
|
+
#
|
51
|
+
# @option options [String, Regexp] :host
|
52
|
+
# The host-name to visit.
|
53
|
+
#
|
54
|
+
# @option options [Array] :hosts
|
55
|
+
# Patterns of host-names to visit.
|
56
|
+
#
|
57
|
+
# @option options [Array] :ignore_hosts
|
58
|
+
# Patterns of host-names not to visit.
|
59
|
+
#
|
60
|
+
# @option options [Array] :ports
|
61
|
+
# Ports to visit.
|
62
|
+
#
|
63
|
+
# @option options [Array] :ignore_ports
|
64
|
+
# Ports not to visit.
|
65
|
+
#
|
66
|
+
# @option options [Array] :links
|
67
|
+
# Patterns of links to visit.
|
68
|
+
#
|
69
|
+
# @option options [Array] :ignore_links
|
70
|
+
# Patterns of links not to visit.
|
71
|
+
#
|
72
|
+
# @option options [Array] :ext
|
73
|
+
# Patterns of file extensions to accept.
|
74
|
+
#
|
75
|
+
# @option options [Array] :ignore_exts
|
76
|
+
# Patterns of file extensions to ignore.
|
77
|
+
#
|
78
|
+
# @yield [spider]
|
79
|
+
# If a block is given, it will be passed the newly created spider.
|
80
|
+
#
|
81
|
+
# @yieldparam [Spider] spider
|
82
|
+
# The newly created spider.
|
83
|
+
#
|
84
|
+
# @see http://spidr.rubyforge.org/docs/classes/Spidr/Agent.html
|
58
85
|
#
|
59
86
|
def initialize(options={},&block)
|
60
87
|
options = {
|
data/lib/ronin/web/version.rb
CHANGED
@@ -1,5 +1,4 @@
|
|
1
1
|
#
|
2
|
-
#--
|
3
2
|
# Ronin Web - A Ruby library for Ronin that provides support for web
|
4
3
|
# scraping and spidering functionality.
|
5
4
|
#
|
@@ -18,12 +17,11 @@
|
|
18
17
|
# You should have received a copy of the GNU General Public License
|
19
18
|
# along with this program; if not, write to the Free Software
|
20
19
|
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
|
21
|
-
#++
|
22
20
|
#
|
23
21
|
|
24
22
|
module Ronin
|
25
23
|
module Web
|
26
24
|
# Ronin Web Version
|
27
|
-
VERSION = '0.
|
25
|
+
VERSION = '0.2.0'
|
28
26
|
end
|
29
27
|
end
|
data/lib/ronin/web/web.rb
CHANGED
@@ -1,5 +1,4 @@
|
|
1
1
|
#
|
2
|
-
#--
|
3
2
|
# Ronin Web - A Ruby library for Ronin that provides support for web
|
4
3
|
# scraping and spidering functionality.
|
5
4
|
#
|
@@ -18,7 +17,6 @@
|
|
18
17
|
# You should have received a copy of the GNU General Public License
|
19
18
|
# along with this program; if not, write to the Free Software
|
20
19
|
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
|
21
|
-
#++
|
22
20
|
#
|
23
21
|
|
24
22
|
require 'ronin/network/http'
|
@@ -31,8 +29,19 @@ require 'open-uri'
|
|
31
29
|
module Ronin
|
32
30
|
module Web
|
33
31
|
#
|
34
|
-
#
|
35
|
-
#
|
32
|
+
# Parses the body of a document into a HTML document object.
|
33
|
+
#
|
34
|
+
# @param [String, IO] body
|
35
|
+
# The body of the document to parse.
|
36
|
+
#
|
37
|
+
# @yield [doc]
|
38
|
+
# If a block is given, it will be passed the newly created document
|
39
|
+
# object.
|
40
|
+
#
|
41
|
+
# @yieldparam [Nokogiri::HTML::Document] doc
|
42
|
+
# The new HTML document object.
|
43
|
+
#
|
44
|
+
# @return [Nokogiri::HTML::Document] The new HTML document object.
|
36
45
|
#
|
37
46
|
def Web.html(body,&block)
|
38
47
|
doc = Nokogiri::HTML(body)
|
@@ -42,8 +51,15 @@ module Ronin
|
|
42
51
|
end
|
43
52
|
|
44
53
|
#
|
45
|
-
# Creates a new Nokogiri::HTML::Builder
|
54
|
+
# Creates a new Nokogiri::HTML::Builder.
|
55
|
+
#
|
56
|
+
# @yield []
|
57
|
+
# The block that will be used to construct the HTML document.
|
46
58
|
#
|
59
|
+
# @return [Nokogiri::HTML::Builder]
|
60
|
+
# The new HTML builder object.
|
61
|
+
#
|
62
|
+
# @example
|
47
63
|
# Web.build_html do
|
48
64
|
# body {
|
49
65
|
# div(:style => 'display:none;') {
|
@@ -57,8 +73,19 @@ module Ronin
|
|
57
73
|
end
|
58
74
|
|
59
75
|
#
|
60
|
-
#
|
61
|
-
#
|
76
|
+
# Parses the body of a document into a XML document object.
|
77
|
+
#
|
78
|
+
# @param [String, IO] body
|
79
|
+
# The body of the document to parse.
|
80
|
+
#
|
81
|
+
# @yield [doc]
|
82
|
+
# If a block is given, it will be passed the newly created document
|
83
|
+
# object.
|
84
|
+
#
|
85
|
+
# @yieldparam [Nokogiri::XML::Document] doc
|
86
|
+
# The new XML document object.
|
87
|
+
#
|
88
|
+
# @return [Nokogiri::XML::Document] The new XML document object.
|
62
89
|
#
|
63
90
|
def Web.xml(body,&block)
|
64
91
|
doc = Nokogiri::XML(body)
|
@@ -68,8 +95,15 @@ module Ronin
|
|
68
95
|
end
|
69
96
|
|
70
97
|
#
|
71
|
-
# Creates a new Nokogiri::XML::Builder
|
98
|
+
# Creates a new Nokogiri::XML::Builder.
|
72
99
|
#
|
100
|
+
# @yield []
|
101
|
+
# The block that will be used to construct the XML document.
|
102
|
+
#
|
103
|
+
# @return [Nokogiri::XML::Builder]
|
104
|
+
# The new XML builder object.
|
105
|
+
#
|
106
|
+
# @example
|
73
107
|
# Web.build_xml do
|
74
108
|
# post(:id => 2) {
|
75
109
|
# title { text("some example) }
|
@@ -82,104 +116,113 @@ module Ronin
|
|
82
116
|
end
|
83
117
|
|
84
118
|
#
|
85
|
-
#
|
119
|
+
# Proxy information for Ronin::Web to use.
|
86
120
|
#
|
87
|
-
|
88
|
-
|
89
|
-
end
|
90
|
-
|
121
|
+
# @return [Network::HTTP::Proxy]
|
122
|
+
# The Ronin Web proxy information.
|
91
123
|
#
|
92
|
-
#
|
93
|
-
#
|
94
|
-
def Web.default_proxy_port=(port)
|
95
|
-
Network::HTTP.default_proxy_port = port
|
96
|
-
end
|
97
|
-
|
98
|
-
#
|
99
|
-
# Returns the +Hash+ of the Ronin Web proxy information.
|
124
|
+
# @see Ronin::Network::HTTP.proxy
|
100
125
|
#
|
101
126
|
def Web.proxy
|
102
127
|
Network::HTTP.proxy
|
103
128
|
end
|
104
129
|
|
105
130
|
#
|
106
|
-
#
|
131
|
+
# Creates a HTTP URI based on a Hash of proxy information.
|
107
132
|
#
|
108
|
-
|
109
|
-
|
110
|
-
end
|
111
|
-
|
133
|
+
# @param [Network::HTTP::Proxy, Hash] proxy_info
|
134
|
+
# The proxy information.
|
112
135
|
#
|
113
|
-
#
|
114
|
-
#
|
136
|
+
# @return [URI::HTTP, nil]
|
137
|
+
# The HTTP URI that represents the proxy. If the proxy is diabled,
|
138
|
+
# +nil+ will be returned.
|
115
139
|
#
|
116
140
|
def Web.proxy_url(proxy_info=Web.proxy)
|
117
|
-
|
118
|
-
userinfo = nil
|
119
|
-
|
120
|
-
if (Web.proxy[:user] || Web.proxy[:password])
|
121
|
-
userinfo = "#{Web.proxy[:user]}:#{Web.proxy[:password]}"
|
122
|
-
end
|
123
|
-
|
124
|
-
return URI::HTTP.build(
|
125
|
-
:host => Web.proxy[:host],
|
126
|
-
:port => Web.proxy[:port],
|
127
|
-
:userinfo => userinfo,
|
128
|
-
:path => '/'
|
129
|
-
)
|
130
|
-
end
|
141
|
+
Network::HTTP::Proxy.new(proxy_info).url
|
131
142
|
end
|
132
143
|
|
133
144
|
#
|
134
|
-
#
|
145
|
+
# @return [Array]
|
146
|
+
# The supported Web User-Agent Aliases.
|
135
147
|
#
|
136
148
|
def Web.user_agent_aliases
|
137
149
|
WWW::Mechanize::AGENT_ALIASES
|
138
150
|
end
|
139
151
|
|
140
152
|
#
|
141
|
-
#
|
153
|
+
# @return [String, nil]
|
154
|
+
# The Ronin Web User-Agent
|
142
155
|
#
|
143
156
|
def Web.user_agent
|
144
157
|
Network::HTTP.user_agent
|
145
158
|
end
|
146
159
|
|
147
160
|
#
|
148
|
-
# Sets the Ronin Web User-Agent
|
161
|
+
# Sets the Ronin Web User-Agent.
|
162
|
+
#
|
163
|
+
# @param [String] new_agent
|
164
|
+
# The User-Agent string to use.
|
165
|
+
#
|
166
|
+
# @return [String]
|
167
|
+
# The new User-Agent string.
|
149
168
|
#
|
150
169
|
def Web.user_agent=(new_agent)
|
151
170
|
Network::HTTP.user_agent = new_agent
|
152
171
|
end
|
153
172
|
|
154
173
|
#
|
155
|
-
# Sets the Ronin Web User-Agent
|
156
|
-
#
|
174
|
+
# Sets the Ronin Web User-Agent.
|
175
|
+
#
|
176
|
+
# @param [String] name
|
177
|
+
# The User-Agent alias to use.
|
178
|
+
#
|
179
|
+
# @return [String]
|
180
|
+
# The new User-Agent string.
|
157
181
|
#
|
158
182
|
def Web.user_agent_alias=(name)
|
159
183
|
Network::HTTP.user_agent = Web.user_agent_aliases[name.to_s]
|
160
184
|
end
|
161
185
|
|
162
186
|
#
|
163
|
-
# Opens
|
164
|
-
#
|
187
|
+
# Opens a URL as a temporary file.
|
188
|
+
#
|
189
|
+
# @param [Hash] options
|
190
|
+
# Additional options.
|
191
|
+
#
|
192
|
+
# @option options [String] :user_agent_alias
|
193
|
+
# The User-Agent Alias to use.
|
194
|
+
#
|
195
|
+
# @option options [String] :user_agent
|
196
|
+
# The User-Agent string to use.
|
165
197
|
#
|
166
|
-
#
|
167
|
-
#
|
168
|
-
# <tt>:user_agent</tt>:: The User-Agent string to use.
|
169
|
-
# <tt>:proxy</tt>:: A +Hash+ of the proxy information to use.
|
170
|
-
# <tt>:user</tt>:: The HTTP Basic Authentication user name.
|
171
|
-
# <tt>:password</tt>:: The HTTP Basic Authentication password.
|
172
|
-
# <tt>:content_length_proc</tt>:: A callback which will be passed the
|
173
|
-
# content-length of the HTTP response.
|
174
|
-
# <tt>:progress_proc</tt>:: A callback which will be passed the size
|
175
|
-
# of each fragment, once received from the
|
176
|
-
# server.
|
198
|
+
# @option options [Network::HTTP::Proxy, Hash] :proxy (Web.proxy)
|
199
|
+
# Proxy information.
|
177
200
|
#
|
201
|
+
# @option options [String] :user
|
202
|
+
# The HTTP Basic Authentication user name.
|
203
|
+
#
|
204
|
+
# @option options [String] :password
|
205
|
+
# The HTTP Basic Authentication password.
|
206
|
+
#
|
207
|
+
# @option options [Proc] :content_length_proc
|
208
|
+
# A callback which will be passed the content-length of the HTTP
|
209
|
+
# response.
|
210
|
+
#
|
211
|
+
# @option options [Proc] :progress_proc
|
212
|
+
# A callback which will be passed the size of each fragment, once
|
213
|
+
# received from the server.
|
214
|
+
#
|
215
|
+
# @return [File]
|
216
|
+
# The contents of the URL.
|
217
|
+
#
|
218
|
+
# @example Open a given URL.
|
178
219
|
# Web.open('http://www.hackety.org/')
|
179
220
|
#
|
221
|
+
# @example Open a given URL, using a custom User-Agent alias.
|
180
222
|
# Web.open('http://tenderlovemaking.com/',
|
181
223
|
# :user_agent_alias => 'Linux Mozilla')
|
182
224
|
#
|
225
|
+
# @example Open a given URL, using a custom User-Agent string.
|
183
226
|
# Web.open('http://www.wired.com/', :user_agent => 'the future')
|
184
227
|
#
|
185
228
|
def Web.open(url,options={})
|
@@ -216,17 +259,41 @@ module Ronin
|
|
216
259
|
end
|
217
260
|
|
218
261
|
#
|
219
|
-
# Creates a new Mechanize
|
262
|
+
# Creates a new Mechanize Agent.
|
263
|
+
#
|
264
|
+
# @param [Hash] options
|
265
|
+
# Additional options.
|
266
|
+
#
|
267
|
+
# @option options [String] :user_agent_alias
|
268
|
+
# The User-Agent Alias to use.
|
269
|
+
#
|
270
|
+
# @option options [String] :user_agent
|
271
|
+
# The User-Agent string to use.
|
272
|
+
#
|
273
|
+
# @option options [Network::HTTP::Proxy, Hash] :proxy (Web.proxy)
|
274
|
+
# Proxy information.
|
220
275
|
#
|
221
|
-
#
|
222
|
-
#
|
223
|
-
#
|
224
|
-
# <tt>:proxy</tt>:: A +Hash+ of the proxy information to use.
|
276
|
+
# @yield [agent]
|
277
|
+
# If a block is given, it will be passed the newly created Mechanize
|
278
|
+
# agent.
|
225
279
|
#
|
280
|
+
# @yieldparam [WWW::Mechanize] agent
|
281
|
+
# The new Mechanize agent.
|
282
|
+
#
|
283
|
+
# @return [WWW::Mechanize]
|
284
|
+
# The new Mechanize agent.
|
285
|
+
#
|
286
|
+
# @example Create a new agent.
|
226
287
|
# Web.agent
|
288
|
+
#
|
289
|
+
# @example Create a new agent, with a custom User-Agent alias.
|
227
290
|
# Web.agent(:user_agent_alias => 'Linux Mozilla')
|
291
|
+
#
|
292
|
+
# @example Create a new agent, with a custom User-Agent string.
|
228
293
|
# Web.agent(:user_agent => 'wooden pants')
|
229
294
|
#
|
295
|
+
# @see http://mechanize.rubyforge.org/mechanize/WWW/Mechanize.html
|
296
|
+
#
|
230
297
|
def Web.agent(options={},&block)
|
231
298
|
agent = WWW::Mechanize.new
|
232
299
|
|
@@ -248,22 +315,46 @@ module Ronin
|
|
248
315
|
end
|
249
316
|
|
250
317
|
#
|
251
|
-
#
|
252
|
-
#
|
318
|
+
# Creates a Mechanize Page for the contents at a given URL.
|
319
|
+
#
|
320
|
+
# @param [URI::Generic, String] url
|
321
|
+
# The URL to request.
|
322
|
+
#
|
323
|
+
# @param [Hash] options
|
324
|
+
# Additional options.
|
325
|
+
#
|
326
|
+
# @option options [String] :user_agent_alias
|
327
|
+
# The User-Agent Alias to use.
|
328
|
+
#
|
329
|
+
# @option options [String] :user_agent
|
330
|
+
# The User-Agent string to use.
|
253
331
|
#
|
254
|
-
#
|
255
|
-
#
|
256
|
-
# <tt>:user_agent</tt>:: The User-Agent string to use.
|
257
|
-
# <tt>:proxy</tt>:: A +Hash+ of the proxy information to use.
|
332
|
+
# @option options [Network::HTTP::Proxy, Hash] :proxy (Web.proxy)
|
333
|
+
# Proxy information.
|
258
334
|
#
|
259
|
-
#
|
335
|
+
# @yield [page]
|
336
|
+
# If a block is given, it will be passed the page for the requested
|
337
|
+
# URL.
|
260
338
|
#
|
339
|
+
# @yieldparam [WWW::Mechanize::Page] page
|
340
|
+
# The requested page.
|
341
|
+
#
|
342
|
+
# @return [WWW::Mechanize::Page]
|
343
|
+
# The requested page.
|
344
|
+
#
|
345
|
+
# @example
|
346
|
+
# Web.get('http://www.0x000000.com')
|
347
|
+
# # => WWW::Mechanize::Page
|
348
|
+
#
|
349
|
+
# @example
|
261
350
|
# Web.get('http://www.rubyinside.com') do |page|
|
262
351
|
# page.search('div.post/h2/a').each do |title|
|
263
352
|
# puts title.inner_text
|
264
353
|
# end
|
265
354
|
# end
|
266
355
|
#
|
356
|
+
# @see http://mechanize.rubyforge.org/mechanize/WWW/Mechanize/Page.html
|
357
|
+
#
|
267
358
|
def Web.get(url,options={},&block)
|
268
359
|
page = Web.agent(options).get(url)
|
269
360
|
|
@@ -272,17 +363,37 @@ module Ronin
|
|
272
363
|
end
|
273
364
|
|
274
365
|
#
|
275
|
-
#
|
276
|
-
# of the
|
277
|
-
#
|
366
|
+
# Requests the body of the Mechanize Page created from the response
|
367
|
+
# of the given URL.
|
368
|
+
#
|
369
|
+
# @param [URI::Generic, String] url
|
370
|
+
# The URL to request.
|
371
|
+
#
|
372
|
+
# @param [Hash] options
|
373
|
+
# Additional options.
|
374
|
+
#
|
375
|
+
# @option options [String] :user_agent_alias
|
376
|
+
# The User-Agent Alias to use.
|
377
|
+
#
|
378
|
+
# @option options [String] :user_agent
|
379
|
+
# The User-Agent string to use.
|
380
|
+
#
|
381
|
+
# @option options [Network::HTTP::Proxy, Hash] :proxy (Web.proxy)
|
382
|
+
# Proxy information.
|
278
383
|
#
|
279
|
-
#
|
280
|
-
#
|
281
|
-
# <tt>:user_agent</tt>:: The User-Agent string to use.
|
282
|
-
# <tt>:proxy</tt>:: A +Hash+ of the proxy information to use.
|
384
|
+
# @yield [body]
|
385
|
+
# If a block is given, it will be passed the body of the page.
|
283
386
|
#
|
387
|
+
# @yieldparam [String] body
|
388
|
+
# The requested body of the page.
|
389
|
+
#
|
390
|
+
# @return [String]
|
391
|
+
# The requested body of the page.
|
392
|
+
#
|
393
|
+
# @example
|
284
394
|
# Web.get_body('http://www.rubyinside.com') # => String
|
285
395
|
#
|
396
|
+
# @example
|
286
397
|
# Web.get_body('http://www.rubyinside.com') do |body|
|
287
398
|
# puts body
|
288
399
|
# end
|
@@ -295,16 +406,39 @@ module Ronin
|
|
295
406
|
end
|
296
407
|
|
297
408
|
#
|
298
|
-
# Posts
|
299
|
-
#
|
409
|
+
# Posts to a given URL and creates a Mechanize Page from the response.
|
410
|
+
#
|
411
|
+
# @param [URI::Generic, String] url
|
412
|
+
# The URL to request.
|
413
|
+
#
|
414
|
+
# @param [Hash] options
|
415
|
+
# Additional options.
|
416
|
+
#
|
417
|
+
# @option options [Hash] :query
|
418
|
+
# Additional query parameters to post with.
|
419
|
+
#
|
420
|
+
# @option options [String] :user_agent_alias
|
421
|
+
# The User-Agent Alia to use.
|
422
|
+
#
|
423
|
+
# @option options [String] :user_agent
|
424
|
+
# The User-Agent string to use.
|
425
|
+
#
|
426
|
+
# @option options [Network::HTTP::Proxy, Hash] :proxy (Web.proxy)
|
427
|
+
# Proxy information.
|
428
|
+
#
|
429
|
+
# @yield [page]
|
430
|
+
# If a block is given, it will be passed the page for the requested
|
431
|
+
# URL.
|
300
432
|
#
|
301
|
-
#
|
302
|
-
#
|
303
|
-
# <tt>:user_agent_alias</tt>:: The User-Agent Alias to use.
|
304
|
-
# <tt>:user_agent</tt>:: The User-Agent string to use.
|
305
|
-
# <tt>:proxy</tt>:: A +Hash+ of the proxy information to use.
|
433
|
+
# @yieldparam [WWW::Mechanize::Page] page
|
434
|
+
# The requested page.
|
306
435
|
#
|
307
|
-
#
|
436
|
+
# @return [WWW::Mechanize::Page]
|
437
|
+
# The requested page.
|
438
|
+
#
|
439
|
+
# @example
|
440
|
+
# Web.post('http://www.rubyinside.com')
|
441
|
+
# # => WWW::Mechanize::Page
|
308
442
|
#
|
309
443
|
def Web.post(url,options={},&block)
|
310
444
|
query = {}
|
@@ -317,17 +451,41 @@ module Ronin
|
|
317
451
|
end
|
318
452
|
|
319
453
|
#
|
320
|
-
#
|
321
|
-
#
|
322
|
-
#
|
454
|
+
# Posts to a given URL and returns the body of the Mechanize Page
|
455
|
+
# created from the response.
|
456
|
+
#
|
457
|
+
# @param [URI::Generic, String] url
|
458
|
+
# The URL to request.
|
459
|
+
#
|
460
|
+
# @param [Hash] options
|
461
|
+
# Additional options.
|
462
|
+
#
|
463
|
+
# @option options [Hash] :query
|
464
|
+
# Additional query parameters to post with.
|
465
|
+
#
|
466
|
+
# @option options [String] :user_agent_alias
|
467
|
+
# The User-Agent Alias to use.
|
468
|
+
#
|
469
|
+
# @option options [String] :user_agent
|
470
|
+
# The User-Agent string to use.
|
471
|
+
#
|
472
|
+
# @option options [Network::HTTP::Proxy, Hash] :proxy (Web.proxy)
|
473
|
+
# Proxy information.
|
474
|
+
#
|
475
|
+
# @yield [body]
|
476
|
+
# If a block is given, it will be passed the body of the page.
|
477
|
+
#
|
478
|
+
# @yieldparam [WWW::Mechanize::Page] page
|
479
|
+
# The body of the requested page.
|
323
480
|
#
|
324
|
-
#
|
325
|
-
#
|
326
|
-
# <tt>:user_agent</tt>:: The User-Agent string to use.
|
327
|
-
# <tt>:proxy</tt>:: A +Hash+ of the proxy information to use.
|
481
|
+
# @return [WWW::Mechanize::Page]
|
482
|
+
# The body of the requested page.
|
328
483
|
#
|
329
|
-
#
|
484
|
+
# @example
|
485
|
+
# Web.post_body('http://www.rubyinside.com')
|
486
|
+
# # => String
|
330
487
|
#
|
488
|
+
# @example
|
331
489
|
# Web.post_body('http://www.rubyinside.com') do |body|
|
332
490
|
# puts body
|
333
491
|
# end
|